289b61e5239d58a8b72c1ab88d46a29712b933fc
[dpdk.git] / drivers / net / sfc / sfc_ef10_essb_rx.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  *
3  * Copyright (c) 2017-2018 Solarflare Communications Inc.
4  * All rights reserved.
5  *
6  * This software was jointly developed between OKTET Labs (under contract
7  * for Solarflare) and Solarflare Communications, Inc.
8  */
9
10 /* EF10 equal stride packed stream receive native datapath implementation */
11
12 #include <stdbool.h>
13
14 #include <rte_byteorder.h>
15 #include <rte_mbuf_ptype.h>
16 #include <rte_mbuf.h>
17 #include <rte_io.h>
18
19 #include "efx.h"
20 #include "efx_types.h"
21 #include "efx_regs.h"
22 #include "efx_regs_ef10.h"
23
24 #include "sfc_tweak.h"
25 #include "sfc_dp_rx.h"
26 #include "sfc_kvargs.h"
27 #include "sfc_ef10.h"
28
29 /* Tunnels are not supported */
30 #define SFC_EF10_RX_EV_ENCAP_SUPPORT    0
31 #include "sfc_ef10_rx_ev.h"
32
33 #define sfc_ef10_essb_rx_err(dpq, ...) \
34         SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10_ESSB, ERR, dpq, __VA_ARGS__)
35
36 #define sfc_ef10_essb_rx_info(dpq, ...) \
37         SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10_ESSB, INFO, dpq, __VA_ARGS__)
38
39 /*
40  * Fake length for RXQ descriptors in equal stride super-buffer mode
41  * to make hardware happy.
42  */
43 #define SFC_EF10_ESSB_RX_FAKE_BUF_SIZE  32
44
45 /**
46  * Minimum number of Rx buffers the datapath allows to use.
47  *
48  * Each HW Rx descriptor has many Rx buffers. The number of buffers
49  * in one HW Rx descriptor is equal to size of contiguous block
50  * provided by Rx buffers memory pool. The contiguous block size
51  * depends on CONFIG_RTE_DRIVER_MEMPOOL_BUCKET_SIZE_KB and rte_mbuf
52  * data size specified on the memory pool creation. Typical rte_mbuf
53  * data size is about 2k which makes a bit less than 32 buffers in
54  * contiguous block with default bucket size equal to 64k.
55  * Since HW Rx descriptors are pushed by 8 (see SFC_EF10_RX_WPTR_ALIGN),
56  * it makes about 256 as required minimum. Double it in advertised
57  * minimum to allow for at least 2 refill blocks.
58  */
59 #define SFC_EF10_ESSB_RX_DESCS_MIN      512
60
61 /**
62  * Number of Rx buffers should be aligned to.
63  *
64  * There are no extra requirements on alignment since actual number of
65  * pushed Rx buffers will be multiple by contiguous block size which
66  * is unknown beforehand.
67  */
68 #define SFC_EF10_ESSB_RX_DESCS_ALIGN    1
69
70 /**
71  * Maximum number of descriptors/buffers in the Rx ring.
72  * It should guarantee that corresponding event queue never overfill.
73  */
74 #define SFC_EF10_ESSB_RXQ_LIMIT(_nevs) \
75         ((_nevs) - 1 /* head must not step on tail */ - \
76          (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \
77          1 /* Rx error */ - 1 /* flush */)
78
79 struct sfc_ef10_essb_rx_sw_desc {
80         struct rte_mbuf                 *first_mbuf;
81 };
82
83 struct sfc_ef10_essb_rxq {
84         /* Used on data path */
85         unsigned int                    flags;
86 #define SFC_EF10_ESSB_RXQ_STARTED       0x1
87 #define SFC_EF10_ESSB_RXQ_NOT_RUNNING   0x2
88 #define SFC_EF10_ESSB_RXQ_EXCEPTION     0x4
89         unsigned int                    rxq_ptr_mask;
90         unsigned int                    block_size;
91         unsigned int                    buf_stride;
92         unsigned int                    bufs_ptr;
93         unsigned int                    completed;
94         unsigned int                    pending_id;
95         unsigned int                    bufs_pending;
96         unsigned int                    left_in_completed;
97         unsigned int                    left_in_pending;
98         unsigned int                    evq_read_ptr;
99         unsigned int                    evq_ptr_mask;
100         efx_qword_t                     *evq_hw_ring;
101         struct sfc_ef10_essb_rx_sw_desc *sw_ring;
102         uint16_t                        port_id;
103
104         /* Used on refill */
105         unsigned int                    added;
106         unsigned int                    max_fill_level;
107         unsigned int                    refill_threshold;
108         struct rte_mempool              *refill_mb_pool;
109         efx_qword_t                     *rxq_hw_ring;
110         volatile void                   *doorbell;
111
112         /* Datapath receive queue anchor */
113         struct sfc_dp_rxq               dp;
114 };
115
116 static inline struct sfc_ef10_essb_rxq *
117 sfc_ef10_essb_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq)
118 {
119         return container_of(dp_rxq, struct sfc_ef10_essb_rxq, dp);
120 }
121
122 static struct rte_mbuf *
123 sfc_ef10_essb_next_mbuf(const struct sfc_ef10_essb_rxq *rxq,
124                         struct rte_mbuf *mbuf)
125 {
126         return (struct rte_mbuf *)((uintptr_t)mbuf + rxq->buf_stride);
127 }
128
129 static struct rte_mbuf *
130 sfc_ef10_essb_mbuf_by_index(const struct sfc_ef10_essb_rxq *rxq,
131                             struct rte_mbuf *mbuf, unsigned int idx)
132 {
133         return (struct rte_mbuf *)((uintptr_t)mbuf + idx * rxq->buf_stride);
134 }
135
136 static struct rte_mbuf *
137 sfc_ef10_essb_maybe_next_completed(struct sfc_ef10_essb_rxq *rxq)
138 {
139         const struct sfc_ef10_essb_rx_sw_desc *rxd;
140
141         if (rxq->left_in_completed != 0) {
142                 rxd = &rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask];
143                 return sfc_ef10_essb_mbuf_by_index(rxq, rxd->first_mbuf,
144                                 rxq->block_size - rxq->left_in_completed);
145         } else {
146                 rxq->completed++;
147                 rxd = &rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask];
148                 rxq->left_in_completed = rxq->block_size;
149                 return rxd->first_mbuf;
150         }
151 }
152
153 static void
154 sfc_ef10_essb_rx_qrefill(struct sfc_ef10_essb_rxq *rxq)
155 {
156         const unsigned int rxq_ptr_mask = rxq->rxq_ptr_mask;
157         unsigned int free_space;
158         unsigned int bulks;
159         void *mbuf_blocks[SFC_EF10_RX_WPTR_ALIGN];
160         unsigned int added = rxq->added;
161
162         free_space = rxq->max_fill_level - (added - rxq->completed);
163
164         if (free_space < rxq->refill_threshold)
165                 return;
166
167         bulks = free_space / RTE_DIM(mbuf_blocks);
168         /* refill_threshold guarantees that bulks is positive */
169         SFC_ASSERT(bulks > 0);
170
171         do {
172                 unsigned int id;
173                 unsigned int i;
174
175                 if (unlikely(rte_mempool_get_contig_blocks(rxq->refill_mb_pool,
176                                 mbuf_blocks, RTE_DIM(mbuf_blocks)) < 0)) {
177                         struct rte_eth_dev_data *dev_data =
178                                 rte_eth_devices[rxq->port_id].data;
179
180                         /*
181                          * It is hardly a safe way to increment counter
182                          * from different contexts, but all PMDs do it.
183                          */
184                         dev_data->rx_mbuf_alloc_failed += RTE_DIM(mbuf_blocks);
185                         /* Return if we have posted nothing yet */
186                         if (added == rxq->added)
187                                 return;
188                         /* Push posted */
189                         break;
190                 }
191
192                 for (i = 0, id = added & rxq_ptr_mask;
193                      i < RTE_DIM(mbuf_blocks);
194                      ++i, ++id) {
195                         struct rte_mbuf *m = mbuf_blocks[i];
196                         struct sfc_ef10_essb_rx_sw_desc *rxd;
197
198                         SFC_ASSERT((id & ~rxq_ptr_mask) == 0);
199                         rxd = &rxq->sw_ring[id];
200                         rxd->first_mbuf = m;
201
202                         /* RX_KER_BYTE_CNT is ignored by firmware */
203                         EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id],
204                                              ESF_DZ_RX_KER_BYTE_CNT,
205                                              SFC_EF10_ESSB_RX_FAKE_BUF_SIZE,
206                                              ESF_DZ_RX_KER_BUF_ADDR,
207                                              rte_mbuf_data_iova_default(m));
208                 }
209
210                 added += RTE_DIM(mbuf_blocks);
211
212         } while (--bulks > 0);
213
214         SFC_ASSERT(rxq->added != added);
215         rxq->added = added;
216         sfc_ef10_rx_qpush(rxq->doorbell, added, rxq_ptr_mask);
217 }
218
219 static bool
220 sfc_ef10_essb_rx_event_get(struct sfc_ef10_essb_rxq *rxq, efx_qword_t *rx_ev)
221 {
222         *rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->evq_ptr_mask];
223
224         if (!sfc_ef10_ev_present(*rx_ev))
225                 return false;
226
227         if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) !=
228                      FSE_AZ_EV_CODE_RX_EV)) {
229                 /*
230                  * Do not move read_ptr to keep the event for exception
231                  * handling
232                  */
233                 rxq->flags |= SFC_EF10_ESSB_RXQ_EXCEPTION;
234                 sfc_ef10_essb_rx_err(&rxq->dp.dpq,
235                                      "RxQ exception at EvQ read ptr %#x",
236                                      rxq->evq_read_ptr);
237                 return false;
238         }
239
240         rxq->evq_read_ptr++;
241         return true;
242 }
243
244 static void
245 sfc_ef10_essb_rx_process_ev(struct sfc_ef10_essb_rxq *rxq, efx_qword_t rx_ev)
246 {
247         unsigned int ready;
248
249         ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) -
250                  rxq->bufs_ptr) &
251                 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
252
253         rxq->bufs_ptr += ready;
254         rxq->bufs_pending += ready;
255
256         SFC_ASSERT(ready > 0);
257         do {
258                 const struct sfc_ef10_essb_rx_sw_desc *rxd;
259                 struct rte_mbuf *m;
260                 unsigned int todo_bufs;
261                 struct rte_mbuf *m0;
262
263                 rxd = &rxq->sw_ring[rxq->pending_id];
264                 m = sfc_ef10_essb_mbuf_by_index(rxq, rxd->first_mbuf,
265                         rxq->block_size - rxq->left_in_pending);
266
267                 if (ready < rxq->left_in_pending) {
268                         todo_bufs = ready;
269                         ready = 0;
270                         rxq->left_in_pending -= todo_bufs;
271                 } else {
272                         todo_bufs = rxq->left_in_pending;
273                         ready -= todo_bufs;
274                         rxq->left_in_pending = rxq->block_size;
275                         if (rxq->pending_id != rxq->rxq_ptr_mask)
276                                 rxq->pending_id++;
277                         else
278                                 rxq->pending_id = 0;
279                 }
280
281                 SFC_ASSERT(todo_bufs > 0);
282                 --todo_bufs;
283
284                 sfc_ef10_rx_ev_to_offloads(rx_ev, m, ~0ull);
285
286                 /* Prefetch pseudo-header */
287                 rte_prefetch0((uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM);
288
289                 m0 = m;
290                 while (todo_bufs-- > 0) {
291                         m = sfc_ef10_essb_next_mbuf(rxq, m);
292                         m->ol_flags = m0->ol_flags;
293                         m->packet_type = m0->packet_type;
294                         /* Prefetch pseudo-header */
295                         rte_prefetch0((uint8_t *)m->buf_addr +
296                                       RTE_PKTMBUF_HEADROOM);
297                 }
298         } while (ready > 0);
299 }
300
301 static unsigned int
302 sfc_ef10_essb_rx_get_pending(struct sfc_ef10_essb_rxq *rxq,
303                              struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
304 {
305         unsigned int n_rx_pkts = 0;
306         unsigned int todo_bufs;
307         struct rte_mbuf *m;
308
309         while ((todo_bufs = RTE_MIN(nb_pkts - n_rx_pkts,
310                                     rxq->bufs_pending)) > 0) {
311                 m = sfc_ef10_essb_maybe_next_completed(rxq);
312
313                 todo_bufs = RTE_MIN(todo_bufs, rxq->left_in_completed);
314
315                 rxq->bufs_pending -= todo_bufs;
316                 rxq->left_in_completed -= todo_bufs;
317
318                 SFC_ASSERT(todo_bufs > 0);
319                 todo_bufs--;
320
321                 do {
322                         const efx_qword_t *qwordp;
323                         uint16_t pkt_len;
324
325                         rx_pkts[n_rx_pkts++] = m;
326
327                         /* Parse pseudo-header */
328                         qwordp = (const efx_qword_t *)
329                                 ((uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM);
330                         pkt_len =
331                                 EFX_QWORD_FIELD(*qwordp,
332                                                 ES_EZ_ESSB_RX_PREFIX_DATA_LEN);
333
334                         m->data_off = RTE_PKTMBUF_HEADROOM +
335                                 ES_EZ_ESSB_RX_PREFIX_LEN;
336                         m->port = rxq->port_id;
337
338                         rte_pktmbuf_pkt_len(m) = pkt_len;
339                         rte_pktmbuf_data_len(m) = pkt_len;
340
341                         m->ol_flags |=
342                                 (PKT_RX_RSS_HASH *
343                                  !!EFX_TEST_QWORD_BIT(*qwordp,
344                                         ES_EZ_ESSB_RX_PREFIX_HASH_VALID_LBN)) |
345                                 (PKT_RX_FDIR_ID *
346                                  !!EFX_TEST_QWORD_BIT(*qwordp,
347                                         ES_EZ_ESSB_RX_PREFIX_MARK_VALID_LBN)) |
348                                 (PKT_RX_FDIR *
349                                  !!EFX_TEST_QWORD_BIT(*qwordp,
350                                         ES_EZ_ESSB_RX_PREFIX_MATCH_FLAG_LBN));
351
352                         /* EFX_QWORD_FIELD converts little-endian to CPU */
353                         m->hash.rss =
354                                 EFX_QWORD_FIELD(*qwordp,
355                                                 ES_EZ_ESSB_RX_PREFIX_HASH);
356                         m->hash.fdir.hi =
357                                 EFX_QWORD_FIELD(*qwordp,
358                                                 ES_EZ_ESSB_RX_PREFIX_MARK);
359
360                         m = sfc_ef10_essb_next_mbuf(rxq, m);
361                 } while (todo_bufs-- > 0);
362         }
363
364         return n_rx_pkts;
365 }
366
367
368 static uint16_t
369 sfc_ef10_essb_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
370                         uint16_t nb_pkts)
371 {
372         struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(rx_queue);
373         const unsigned int evq_old_read_ptr = rxq->evq_read_ptr;
374         uint16_t n_rx_pkts;
375         efx_qword_t rx_ev;
376
377         if (unlikely(rxq->flags & (SFC_EF10_ESSB_RXQ_NOT_RUNNING |
378                                    SFC_EF10_ESSB_RXQ_EXCEPTION)))
379                 return 0;
380
381         n_rx_pkts = sfc_ef10_essb_rx_get_pending(rxq, rx_pkts, nb_pkts);
382
383         while (n_rx_pkts != nb_pkts &&
384                sfc_ef10_essb_rx_event_get(rxq, &rx_ev)) {
385                 /*
386                  * DROP_EVENT is an internal to the NIC, software should
387                  * never see it and, therefore, may ignore it.
388                  */
389
390                 sfc_ef10_essb_rx_process_ev(rxq, rx_ev);
391                 n_rx_pkts += sfc_ef10_essb_rx_get_pending(rxq,
392                                                           rx_pkts + n_rx_pkts,
393                                                           nb_pkts - n_rx_pkts);
394         }
395
396         sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->evq_ptr_mask,
397                            evq_old_read_ptr, rxq->evq_read_ptr);
398
399         /* It is not a problem if we refill in the case of exception */
400         sfc_ef10_essb_rx_qrefill(rxq);
401
402         return n_rx_pkts;
403 }
404
405 static sfc_dp_rx_qdesc_npending_t sfc_ef10_essb_rx_qdesc_npending;
406 static unsigned int
407 sfc_ef10_essb_rx_qdesc_npending(__rte_unused struct sfc_dp_rxq *dp_rxq)
408 {
409         /*
410          * Correct implementation requires EvQ polling and events
411          * processing.
412          */
413         return -ENOTSUP;
414 }
415
416 static sfc_dp_rx_get_dev_info_t sfc_ef10_essb_rx_get_dev_info;
417 static void
418 sfc_ef10_essb_rx_get_dev_info(struct rte_eth_dev_info *dev_info)
419 {
420         /*
421          * Number of descriptors just defines maximum number of pushed
422          * descriptors (fill level).
423          */
424         dev_info->rx_desc_lim.nb_min = SFC_EF10_ESSB_RX_DESCS_MIN;
425         dev_info->rx_desc_lim.nb_align = SFC_EF10_ESSB_RX_DESCS_ALIGN;
426 }
427
428 static sfc_dp_rx_pool_ops_supported_t sfc_ef10_essb_rx_pool_ops_supported;
429 static int
430 sfc_ef10_essb_rx_pool_ops_supported(const char *pool)
431 {
432         SFC_ASSERT(pool != NULL);
433
434         if (strcmp(pool, "bucket") == 0)
435                 return 0;
436
437         return -ENOTSUP;
438 }
439
440 static sfc_dp_rx_qsize_up_rings_t sfc_ef10_essb_rx_qsize_up_rings;
441 static int
442 sfc_ef10_essb_rx_qsize_up_rings(uint16_t nb_rx_desc,
443                                 struct rte_mempool *mb_pool,
444                                 unsigned int *rxq_entries,
445                                 unsigned int *evq_entries,
446                                 unsigned int *rxq_max_fill_level)
447 {
448         int rc;
449         struct rte_mempool_info mp_info;
450         unsigned int nb_hw_rx_desc;
451         unsigned int max_events;
452
453         rc = rte_mempool_ops_get_info(mb_pool, &mp_info);
454         if (rc != 0)
455                 return -rc;
456         if (mp_info.contig_block_size == 0)
457                 return EINVAL;
458
459         /*
460          * Calculate required number of hardware Rx descriptors each
461          * carrying contig block size Rx buffers.
462          * It cannot be less than Rx write pointer alignment plus 1
463          * in order to avoid cases when the ring is guaranteed to be
464          * empty.
465          */
466         nb_hw_rx_desc = RTE_MAX(SFC_DIV_ROUND_UP(nb_rx_desc,
467                                                  mp_info.contig_block_size),
468                                 SFC_EF10_RX_WPTR_ALIGN + 1);
469         if (nb_hw_rx_desc <= EFX_RXQ_MINNDESCS) {
470                 *rxq_entries = EFX_RXQ_MINNDESCS;
471         } else {
472                 *rxq_entries = rte_align32pow2(nb_hw_rx_desc);
473                 if (*rxq_entries > EFX_RXQ_MAXNDESCS)
474                         return EINVAL;
475         }
476
477         max_events = RTE_ALIGN_FLOOR(nb_hw_rx_desc, SFC_EF10_RX_WPTR_ALIGN) *
478                 mp_info.contig_block_size +
479                 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ +
480                 1 /* Rx error */ + 1 /* flush */ + 1 /* head-tail space */;
481
482         *evq_entries = rte_align32pow2(max_events);
483         *evq_entries = RTE_MAX(*evq_entries, (unsigned int)EFX_EVQ_MINNEVS);
484         *evq_entries = RTE_MIN(*evq_entries, (unsigned int)EFX_EVQ_MAXNEVS);
485
486         /*
487          * May be even maximum event queue size is insufficient to handle
488          * so many Rx descriptors. If so, we should limit Rx queue fill level.
489          */
490         *rxq_max_fill_level = RTE_MIN(nb_rx_desc,
491                                       SFC_EF10_ESSB_RXQ_LIMIT(*evq_entries));
492         return 0;
493 }
494
495 static sfc_dp_rx_qcreate_t sfc_ef10_essb_rx_qcreate;
496 static int
497 sfc_ef10_essb_rx_qcreate(uint16_t port_id, uint16_t queue_id,
498                          const struct rte_pci_addr *pci_addr, int socket_id,
499                          const struct sfc_dp_rx_qcreate_info *info,
500                          struct sfc_dp_rxq **dp_rxqp)
501 {
502         struct rte_mempool * const mp = info->refill_mb_pool;
503         struct rte_mempool_info mp_info;
504         struct sfc_ef10_essb_rxq *rxq;
505         int rc;
506
507         rc = rte_mempool_ops_get_info(mp, &mp_info);
508         if (rc != 0) {
509                 /* Positive errno is used in the driver */
510                 rc = -rc;
511                 goto fail_get_contig_block_size;
512         }
513
514         /* Check if the mempool provides block dequeue */
515         rc = EINVAL;
516         if (mp_info.contig_block_size == 0)
517                 goto fail_no_block_dequeue;
518
519         rc = ENOMEM;
520         rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq),
521                                  RTE_CACHE_LINE_SIZE, socket_id);
522         if (rxq == NULL)
523                 goto fail_rxq_alloc;
524
525         sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr);
526
527         rc = ENOMEM;
528         rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring",
529                                          info->rxq_entries,
530                                          sizeof(*rxq->sw_ring),
531                                          RTE_CACHE_LINE_SIZE, socket_id);
532         if (rxq->sw_ring == NULL)
533                 goto fail_desc_alloc;
534
535         rxq->block_size = mp_info.contig_block_size;
536         rxq->buf_stride = mp->header_size + mp->elt_size + mp->trailer_size;
537         rxq->rxq_ptr_mask = info->rxq_entries - 1;
538         rxq->evq_ptr_mask = info->evq_entries - 1;
539         rxq->evq_hw_ring = info->evq_hw_ring;
540         rxq->port_id = port_id;
541
542         rxq->max_fill_level = info->max_fill_level / mp_info.contig_block_size;
543         rxq->refill_threshold =
544                 RTE_MAX(info->refill_threshold / mp_info.contig_block_size,
545                         SFC_EF10_RX_WPTR_ALIGN);
546         rxq->refill_mb_pool = mp;
547         rxq->rxq_hw_ring = info->rxq_hw_ring;
548
549         rxq->doorbell = (volatile uint8_t *)info->mem_bar +
550                         ER_DZ_RX_DESC_UPD_REG_OFST +
551                         (info->hw_index << info->vi_window_shift);
552
553         sfc_ef10_essb_rx_info(&rxq->dp.dpq,
554                               "block size is %u, buf stride is %u",
555                               rxq->block_size, rxq->buf_stride);
556         sfc_ef10_essb_rx_info(&rxq->dp.dpq,
557                               "max fill level is %u descs (%u bufs), "
558                               "refill threashold %u descs (%u bufs)",
559                               rxq->max_fill_level,
560                               rxq->max_fill_level * rxq->block_size,
561                               rxq->refill_threshold,
562                               rxq->refill_threshold * rxq->block_size);
563
564         *dp_rxqp = &rxq->dp;
565         return 0;
566
567 fail_desc_alloc:
568         rte_free(rxq);
569
570 fail_rxq_alloc:
571 fail_no_block_dequeue:
572 fail_get_contig_block_size:
573         return rc;
574 }
575
576 static sfc_dp_rx_qdestroy_t sfc_ef10_essb_rx_qdestroy;
577 static void
578 sfc_ef10_essb_rx_qdestroy(struct sfc_dp_rxq *dp_rxq)
579 {
580         struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
581
582         rte_free(rxq->sw_ring);
583         rte_free(rxq);
584 }
585
586 static sfc_dp_rx_qstart_t sfc_ef10_essb_rx_qstart;
587 static int
588 sfc_ef10_essb_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr)
589 {
590         struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
591
592         rxq->evq_read_ptr = evq_read_ptr;
593
594         /* Initialize before refill */
595         rxq->completed = rxq->pending_id = rxq->added = 0;
596         rxq->left_in_completed = rxq->left_in_pending = rxq->block_size;
597         rxq->bufs_ptr = UINT_MAX;
598         rxq->bufs_pending = 0;
599
600         sfc_ef10_essb_rx_qrefill(rxq);
601
602         rxq->flags |= SFC_EF10_ESSB_RXQ_STARTED;
603         rxq->flags &=
604                 ~(SFC_EF10_ESSB_RXQ_NOT_RUNNING | SFC_EF10_ESSB_RXQ_EXCEPTION);
605
606         return 0;
607 }
608
609 static sfc_dp_rx_qstop_t sfc_ef10_essb_rx_qstop;
610 static void
611 sfc_ef10_essb_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr)
612 {
613         struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
614
615         rxq->flags |= SFC_EF10_ESSB_RXQ_NOT_RUNNING;
616
617         *evq_read_ptr = rxq->evq_read_ptr;
618 }
619
620 static sfc_dp_rx_qrx_ev_t sfc_ef10_essb_rx_qrx_ev;
621 static bool
622 sfc_ef10_essb_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id)
623 {
624         __rte_unused struct sfc_ef10_essb_rxq *rxq;
625
626         rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
627         SFC_ASSERT(rxq->flags & SFC_EF10_ESSB_RXQ_NOT_RUNNING);
628
629         /*
630          * It is safe to ignore Rx event since we free all mbufs on
631          * queue purge anyway.
632          */
633
634         return false;
635 }
636
637 static sfc_dp_rx_qpurge_t sfc_ef10_essb_rx_qpurge;
638 static void
639 sfc_ef10_essb_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
640 {
641         struct sfc_ef10_essb_rxq *rxq = sfc_ef10_essb_rxq_by_dp_rxq(dp_rxq);
642         unsigned int i, j;
643         const struct sfc_ef10_essb_rx_sw_desc *rxd;
644         struct rte_mbuf *m;
645
646         if (rxq->completed != rxq->added && rxq->left_in_completed > 0) {
647                 rxd = &rxq->sw_ring[rxq->completed & rxq->rxq_ptr_mask];
648                 m = sfc_ef10_essb_mbuf_by_index(rxq, rxd->first_mbuf,
649                                 rxq->block_size - rxq->left_in_completed);
650                 do {
651                         rxq->left_in_completed--;
652                         rte_mempool_put(rxq->refill_mb_pool, m);
653                         m = sfc_ef10_essb_next_mbuf(rxq, m);
654                 } while (rxq->left_in_completed > 0);
655                 rxq->completed++;
656         }
657
658         for (i = rxq->completed; i != rxq->added; ++i) {
659                 rxd = &rxq->sw_ring[i & rxq->rxq_ptr_mask];
660                 m = rxd->first_mbuf;
661                 for (j = 0; j < rxq->block_size; ++j) {
662                         rte_mempool_put(rxq->refill_mb_pool, m);
663                         m = sfc_ef10_essb_next_mbuf(rxq, m);
664                 }
665         }
666
667         rxq->flags &= ~SFC_EF10_ESSB_RXQ_STARTED;
668 }
669
670 struct sfc_dp_rx sfc_ef10_essb_rx = {
671         .dp = {
672                 .name           = SFC_KVARG_DATAPATH_EF10_ESSB,
673                 .type           = SFC_DP_RX,
674                 .hw_fw_caps     = SFC_DP_HW_FW_CAP_EF10 |
675                                   SFC_DP_HW_FW_CAP_RX_ES_SUPER_BUFFER,
676         },
677         .features               = SFC_DP_RX_FEAT_FLOW_FLAG |
678                                   SFC_DP_RX_FEAT_FLOW_MARK,
679         .get_dev_info           = sfc_ef10_essb_rx_get_dev_info,
680         .pool_ops_supported     = sfc_ef10_essb_rx_pool_ops_supported,
681         .qsize_up_rings         = sfc_ef10_essb_rx_qsize_up_rings,
682         .qcreate                = sfc_ef10_essb_rx_qcreate,
683         .qdestroy               = sfc_ef10_essb_rx_qdestroy,
684         .qstart                 = sfc_ef10_essb_rx_qstart,
685         .qstop                  = sfc_ef10_essb_rx_qstop,
686         .qrx_ev                 = sfc_ef10_essb_rx_qrx_ev,
687         .qpurge                 = sfc_ef10_essb_rx_qpurge,
688         .supported_ptypes_get   = sfc_ef10_supported_ptypes_get,
689         .qdesc_npending         = sfc_ef10_essb_rx_qdesc_npending,
690         .pkt_burst              = sfc_ef10_essb_recv_pkts,
691 };