+ struct sfc_efx_rxq *rxq;
+ unsigned int stop;
+ unsigned int pending_id;
+ unsigned int delta;
+ unsigned int i;
+ struct sfc_efx_rx_sw_desc *rxd;
+
+ if (unlikely(evq->exception))
+ goto done;
+
+ rxq = sfc_efx_rxq_by_dp_rxq(evq->dp_rxq);
+
+ SFC_ASSERT(rxq != NULL);
+ SFC_ASSERT(rxq->evq == evq);
+ SFC_ASSERT(rxq->flags & SFC_EFX_RXQ_FLAG_STARTED);
+
+ stop = (id + 1) & rxq->ptr_mask;
+ pending_id = rxq->pending & rxq->ptr_mask;
+ delta = (stop >= pending_id) ? (stop - pending_id) :
+ (rxq->ptr_mask + 1 - pending_id + stop);
+
+ if (delta == 0) {
+ /*
+ * Rx event with no new descriptors done and zero length
+ * is used to abort scattered packet when there is no room
+ * for the tail.
+ */
+ if (unlikely(size != 0)) {
+ evq->exception = B_TRUE;
+ sfc_err(evq->sa,
+ "EVQ %u RxQ %u invalid RX abort "
+ "(id=%#x size=%u flags=%#x); needs restart",
+ evq->evq_index, rxq->dp.dpq.queue_id,
+ id, size, flags);
+ goto done;
+ }
+
+ /* Add discard flag to the first fragment */
+ rxq->sw_desc[pending_id].flags |= EFX_DISCARD;
+ /* Remove continue flag from the last fragment */
+ rxq->sw_desc[id].flags &= ~EFX_PKT_CONT;
+ } else if (unlikely(delta > rxq->batch_max)) {
+ evq->exception = B_TRUE;
+
+ sfc_err(evq->sa,
+ "EVQ %u RxQ %u completion out of order "
+ "(id=%#x delta=%u flags=%#x); needs restart",
+ evq->evq_index, rxq->dp.dpq.queue_id,
+ id, delta, flags);
+
+ goto done;
+ }
+
+ for (i = pending_id; i != stop; i = (i + 1) & rxq->ptr_mask) {
+ rxd = &rxq->sw_desc[i];
+
+ rxd->flags = flags;
+
+ SFC_ASSERT(size < (1 << 16));
+ rxd->size = (uint16_t)size;
+ }
+
+ rxq->pending += delta;
+
+done:
+ return B_FALSE;
+}
+
+static boolean_t
+sfc_ev_dp_rx(void *arg, __rte_unused uint32_t label, uint32_t id,
+ __rte_unused uint32_t size, __rte_unused uint16_t flags)
+{
+ struct sfc_evq *evq = arg;
+ struct sfc_dp_rxq *dp_rxq;
+
+ dp_rxq = evq->dp_rxq;
+ SFC_ASSERT(dp_rxq != NULL);
+
+ SFC_ASSERT(evq->sa->priv.dp_rx->qrx_ev != NULL);
+ return evq->sa->priv.dp_rx->qrx_ev(dp_rxq, id);
+}