X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;ds=sidebyside;f=drivers%2Fcommon%2Fsfc_efx%2Fbase%2Frhead_ev.c;h=2b5635f1cd2e430aab0513f9345b4c2905389333;hb=662286ae61d29fc6fa26ce131f378ac8905eaff9;hp=6113cc9cf3e126b6bddfdfe1665837f04e0399e9;hpb=9edb8ee3e7ab606baea586d8e9179ec61d971456;p=dpdk.git

diff --git a/drivers/common/sfc_efx/base/rhead_ev.c b/drivers/common/sfc_efx/base/rhead_ev.c
index 6113cc9cf3..2b5635f1cd 100644
--- a/drivers/common/sfc_efx/base/rhead_ev.c
+++ b/drivers/common/sfc_efx/base/rhead_ev.c
@@ -16,6 +16,26 @@
  */
 #define	EFX_RHEAD_ALWAYS_INTERRUPTING_EVQ_INDEX	(0)
 
+static			boolean_t
+rhead_ev_dispatch(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eventp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg);
+
+static	__checkReturn	boolean_t
+rhead_ev_rx_packets(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eqp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg);
+
+static	__checkReturn	boolean_t
+rhead_ev_tx_completion(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eqp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg);
 
 static	__checkReturn	boolean_t
 rhead_ev_mcdi(
@@ -24,6 +44,38 @@ rhead_ev_mcdi(
 	__in		const efx_ev_callbacks_t *eecp,
 	__in_opt	void *arg);
 
+#if EFSYS_OPT_EV_EXTENDED_WIDTH
+static			boolean_t
+rhead_ev_ew_dispatch(
+	__in		efx_evq_t *eep,
+	__in		efx_xword_t *eventp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg);
+
+static			void
+rhead_ev_ew_qpoll(
+	__in		efx_evq_t *eep,
+	__inout		unsigned int *countp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg);
+
+#if EFSYS_OPT_DESC_PROXY
+static			boolean_t
+rhead_ev_ew_txq_desc(
+	__in		efx_evq_t *eep,
+	__in		efx_xword_t *eventp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg);
+
+static			boolean_t
+rhead_ev_ew_virtq_desc(
+	__in		efx_evq_t *eep,
+	__in		efx_xword_t *eventp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg);
+#endif /* EFSYS_OPT_DESC_PROXY */
+#endif /* EFSYS_OPT_EV_EXTENDED_WIDTH */
+
 
 	__checkReturn	efx_rc_t
 rhead_ev_init(
@@ -52,18 +104,38 @@ rhead_ev_qcreate(
 	__in		uint32_t flags,
 	__in		efx_evq_t *eep)
 {
+	const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp);
+	size_t desc_size;
 	uint32_t irq;
 	efx_rc_t rc;
 
 	_NOTE(ARGUNUSED(id))	/* buftbl id managed by MC */
 
+	desc_size = encp->enc_ev_desc_size;
+#if EFSYS_OPT_EV_EXTENDED_WIDTH
+	if (flags & EFX_EVQ_FLAGS_EXTENDED_WIDTH)
+		desc_size = encp->enc_ev_ew_desc_size;
+#endif
+	EFSYS_ASSERT(desc_size != 0);
+
+	if (EFSYS_MEM_SIZE(esmp) < (ndescs * desc_size)) {
+		/* Buffer too small for event queue descriptors */
+		rc = EINVAL;
+		goto fail1;
+	}
+
 	/* Set up the handler table */
-	eep->ee_rx	= NULL; /* FIXME */
-	eep->ee_tx	= NULL; /* FIXME */
+	eep->ee_rx	= rhead_ev_rx_packets;
+	eep->ee_tx	= rhead_ev_tx_completion;
 	eep->ee_driver	= NULL; /* FIXME */
 	eep->ee_drv_gen	= NULL; /* FIXME */
 	eep->ee_mcdi	= rhead_ev_mcdi;
 
+#if EFSYS_OPT_DESC_PROXY
+	eep->ee_ew_txq_desc	= rhead_ev_ew_txq_desc;
+	eep->ee_ew_virtq_desc	= rhead_ev_ew_virtq_desc;
+#endif /* EFSYS_OPT_DESC_PROXY */
+
 	/* Set up the event queue */
 	/* INIT_EVQ expects function-relative vector number */
 	if ((flags & EFX_EVQ_FLAGS_NOTIFY_MASK) ==
@@ -84,10 +156,12 @@ rhead_ev_qcreate(
 	rc = efx_mcdi_init_evq(enp, index, esmp, ndescs, irq, us, flags,
 	    B_FALSE);
 	if (rc != 0)
-		goto fail1;
+		goto fail2;
 
 	return (0);
 
+fail2:
+	EFSYS_PROBE(fail2);
 fail1:
 	EFSYS_PROBE1(fail1, efx_rc_t, rc);
 
@@ -119,7 +193,7 @@ rhead_ev_qprime(
 	EFX_POPULATE_DWORD_2(dword, ERF_GZ_EVQ_ID, eep->ee_index,
 	    ERF_GZ_IDX, rptr);
 	/* EVQ_INT_PRIME lives function control window only on Riverhead */
-	EFX_BAR_WRITED(enp, ER_GZ_EVQ_INT_PRIME, &dword, B_FALSE);
+	EFX_BAR_FCW_WRITED(enp, ER_GZ_EVQ_INT_PRIME, &dword);
 
 	return (0);
 }
@@ -135,6 +209,41 @@ rhead_ev_qpost(
 	EFSYS_ASSERT(B_FALSE);
 }
 
+static	__checkReturn	boolean_t
+rhead_ev_dispatch(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eventp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	boolean_t should_abort;
+	uint32_t code;
+
+	code = EFX_QWORD_FIELD(*eventp, ESF_GZ_E_TYPE);
+	switch (code) {
+	case ESE_GZ_EF100_EV_RX_PKTS:
+		should_abort = eep->ee_rx(eep, eventp, eecp, arg);
+		break;
+	case ESE_GZ_EF100_EV_TX_COMPLETION:
+		should_abort = eep->ee_tx(eep, eventp, eecp, arg);
+		break;
+	case ESE_GZ_EF100_EV_MCDI:
+		should_abort = eep->ee_mcdi(eep, eventp, eecp, arg);
+		break;
+	default:
+		EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index,
+		    uint32_t, EFX_QWORD_FIELD(*eventp, EFX_DWORD_1),
+		    uint32_t, EFX_QWORD_FIELD(*eventp, EFX_DWORD_0));
+
+		EFSYS_ASSERT(eecp->eec_exception != NULL);
+		(void) eecp->eec_exception(arg, EFX_EXCEPTION_EV_ERROR, code);
+		should_abort = B_TRUE;
+		break;
+	}
+
+	return (should_abort);
+}
+
 /*
  * Poll event queue in batches. Size of the batch is equal to cache line
  * size divided by event size.
@@ -169,6 +278,13 @@ rhead_ev_qpoll(
 	unsigned int index;
 	size_t offset;
 
+#if EFSYS_OPT_EV_EXTENDED_WIDTH
+	if (eep->ee_flags & EFX_EVQ_FLAGS_EXTENDED_WIDTH) {
+		rhead_ev_ew_qpoll(eep, countp, eecp, arg);
+		return;
+	}
+#endif /* EFSYS_OPT_EV_EXTENDED_WIDTH */
+
 	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
 	EFSYS_ASSERT(countp != NULL);
 	EFSYS_ASSERT(eecp != NULL);
@@ -195,29 +311,12 @@ rhead_ev_qpoll(
 		/* Process the batch of events */
 		for (index = 0; index < total; ++index) {
 			boolean_t should_abort;
-			uint32_t code;
 
 			EFX_EV_QSTAT_INCR(eep, EV_ALL);
 
-			code = EFX_QWORD_FIELD(ev[index], ESF_GZ_E_TYPE);
-			switch (code) {
-			case ESE_GZ_EF100_EV_MCDI:
-				should_abort = eep->ee_mcdi(eep,
-				    &(ev[index]), eecp, arg);
-				break;
-			default:
-				EFSYS_PROBE3(bad_event,
-				    unsigned int, eep->ee_index,
-				    uint32_t,
-				    EFX_QWORD_FIELD(ev[index], EFX_DWORD_1),
-				    uint32_t,
-				    EFX_QWORD_FIELD(ev[index], EFX_DWORD_0));
-
-				EFSYS_ASSERT(eecp->eec_exception != NULL);
-				(void) eecp->eec_exception(arg,
-					EFX_EXCEPTION_EV_ERROR, code);
-				should_abort = B_TRUE;
-			}
+			should_abort =
+			    rhead_ev_dispatch(eep, &(ev[index]), eecp, arg);
+
 			if (should_abort) {
 				/* Ignore subsequent events */
 				total = index + 1;
@@ -246,6 +345,153 @@ rhead_ev_qpoll(
 	*countp = count;
 }
 
+#if EFSYS_OPT_EV_EXTENDED_WIDTH
+static			boolean_t
+rhead_ev_ew_dispatch(
+	__in		efx_evq_t *eep,
+	__in		efx_xword_t *eventp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	boolean_t should_abort;
+	uint32_t code;
+
+	EFSYS_ASSERT((eep->ee_flags & EFX_EVQ_FLAGS_EXTENDED_WIDTH) != 0);
+
+	code = EFX_XWORD_FIELD(*eventp, ESF_GZ_EV_256_EV32_TYPE);
+	switch (code) {
+	case ESE_GZ_EF100_EVEW_64BIT:
+		/* NOTE: ignore phase bit in encapsulated 64bit event. */
+		should_abort =
+		    rhead_ev_dispatch(eep, &eventp->ex_qword[0], eecp, arg);
+		break;
+
+#if EFSYS_OPT_DESC_PROXY
+	case ESE_GZ_EF100_EVEW_TXQ_DESC:
+		should_abort = eep->ee_ew_txq_desc(eep, eventp, eecp, arg);
+		break;
+
+	case ESE_GZ_EF100_EVEW_VIRTQ_DESC:
+		should_abort = eep->ee_ew_virtq_desc(eep, eventp, eecp, arg);
+		break;
+#endif /* EFSYS_OPT_DESC_PROXY */
+
+	default:
+		/* Omit currently unused reserved bits from the probe. */
+		EFSYS_PROBE7(ew_bad_event, unsigned int, eep->ee_index,
+		    uint32_t, EFX_XWORD_FIELD(*eventp, EFX_DWORD_7),
+		    uint32_t, EFX_XWORD_FIELD(*eventp, EFX_DWORD_4),
+		    uint32_t, EFX_XWORD_FIELD(*eventp, EFX_DWORD_3),
+		    uint32_t, EFX_XWORD_FIELD(*eventp, EFX_DWORD_2),
+		    uint32_t, EFX_XWORD_FIELD(*eventp, EFX_DWORD_1),
+		    uint32_t, EFX_XWORD_FIELD(*eventp, EFX_DWORD_0));
+
+		EFSYS_ASSERT(eecp->eec_exception != NULL);
+		(void) eecp->eec_exception(arg, EFX_EXCEPTION_EV_ERROR, code);
+		should_abort = B_TRUE;
+	}
+
+	return (should_abort);
+}
+
+/*
+ * Poll extended width event queue. Size of the batch is equal to cache line
+ * size divided by event size.
+ */
+#define	EF100_EV_EW_BATCH	2
+
+/*
+ * Check if event is present.
+ *
+ * Riverhead EvQs use a phase bit to indicate the presence of valid events,
+ * by flipping the phase bit on each wrap of the write index.
+ */
+#define	EF100_EV_EW_PRESENT(_xword, _phase_bit)				\
+	(EFX_XWORD_FIELD((_xword), ESF_GZ_EV_256_EV32_PHASE) == (_phase_bit))
+
+static			void
+rhead_ev_ew_qpoll(
+	__in		efx_evq_t *eep,
+	__inout		unsigned int *countp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	efx_xword_t ev[EF100_EV_EW_BATCH];
+	unsigned int batch;
+	unsigned int phase_bit;
+	unsigned int total;
+	unsigned int count;
+	unsigned int index;
+	size_t offset;
+
+	EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC);
+	EFSYS_ASSERT((eep->ee_flags & EFX_EVQ_FLAGS_EXTENDED_WIDTH) != 0);
+	EFSYS_ASSERT(countp != NULL);
+	EFSYS_ASSERT(eecp != NULL);
+
+	count = *countp;
+	do {
+		/* Read up until the end of the batch period */
+		batch = EF100_EV_EW_BATCH - (count & (EF100_EV_EW_BATCH - 1));
+		phase_bit = (count & (eep->ee_mask + 1)) != 0;
+		offset = (count & eep->ee_mask) * sizeof (efx_xword_t);
+		for (total = 0; total < batch; ++total) {
+			EFSYS_MEM_READX(eep->ee_esmp, offset, &(ev[total]));
+
+			if (!EF100_EV_EW_PRESENT(ev[total], phase_bit))
+				break;
+
+			/* Omit unused reserved bits from the probe. */
+			EFSYS_PROBE7(ew_event, unsigned int, eep->ee_index,
+			    uint32_t, EFX_XWORD_FIELD(ev[total], EFX_DWORD_7),
+			    uint32_t, EFX_XWORD_FIELD(ev[total], EFX_DWORD_4),
+			    uint32_t, EFX_XWORD_FIELD(ev[total], EFX_DWORD_3),
+			    uint32_t, EFX_XWORD_FIELD(ev[total], EFX_DWORD_2),
+			    uint32_t, EFX_XWORD_FIELD(ev[total], EFX_DWORD_1),
+			    uint32_t, EFX_XWORD_FIELD(ev[total], EFX_DWORD_0));
+
+			offset += sizeof (efx_xword_t);
+		}
+
+		/* Process the batch of events */
+		for (index = 0; index < total; ++index) {
+			boolean_t should_abort;
+
+			EFX_EV_QSTAT_INCR(eep, EV_ALL);
+
+			should_abort =
+			    rhead_ev_ew_dispatch(eep, &(ev[index]), eecp, arg);
+
+			if (should_abort) {
+				/* Ignore subsequent events */
+				total = index + 1;
+
+				/*
+				 * Poison batch to ensure the outer
+				 * loop is broken out of.
+				 */
+				EFSYS_ASSERT(batch <= EF100_EV_EW_BATCH);
+				batch += (EF100_EV_EW_BATCH << 1);
+				EFSYS_ASSERT(total != batch);
+				break;
+			}
+		}
+
+		/*
+		 * There is no necessity to clear processed events since
+		 * phase bit which is flipping on each write index wrap
+		 * is used for event presence indication.
+		 */
+
+		count += total;
+
+	} while (total == batch);
+
+	*countp = count;
+}
+#endif /* EFSYS_OPT_EV_EXTENDED_WIDTH */
+
+
 	__checkReturn	efx_rc_t
 rhead_ev_qmoderate(
 	__in		efx_evq_t *eep,
@@ -274,6 +520,87 @@ rhead_ev_qstats_update(
 }
 #endif /* EFSYS_OPT_QSTATS */
 
+static	__checkReturn	boolean_t
+rhead_ev_rx_packets(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eqp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	efx_nic_t *enp = eep->ee_enp;
+	uint32_t label;
+	uint32_t num_packets;
+	boolean_t should_abort;
+
+	EFX_EV_QSTAT_INCR(eep, EV_RX);
+
+	/* Discard events after RXQ/TXQ errors, or hardware not available */
+	if (enp->en_reset_flags &
+	    (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR | EFX_RESET_HW_UNAVAIL))
+		return (B_FALSE);
+
+	label = EFX_QWORD_FIELD(*eqp, ESF_GZ_EV_RXPKTS_Q_LABEL);
+
+	/*
+	 * On EF100 the EV_RX event reports the number of received
+	 * packets (unlike EF10 which reports a descriptor index).
+	 * The client driver is responsible for maintaining the Rx
+	 * descriptor index, and computing how many descriptors are
+	 * occupied by each received packet (based on the Rx buffer size
+	 * and the packet length from the Rx prefix).
+	 */
+	num_packets = EFX_QWORD_FIELD(*eqp, ESF_GZ_EV_RXPKTS_NUM_PKT);
+
+	/*
+	 * The receive event may indicate more than one packet, and so
+	 * does not contain the packet length. Read the packet length
+	 * from the prefix when handling each packet.
+	 */
+	EFSYS_ASSERT(eecp->eec_rx_packets != NULL);
+	should_abort = eecp->eec_rx_packets(arg, label, num_packets,
+	    EFX_PKT_PREFIX_LEN);
+
+	return (should_abort);
+}
+
+static	__checkReturn	boolean_t
+rhead_ev_tx_completion(
+	__in		efx_evq_t *eep,
+	__in		efx_qword_t *eqp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	efx_nic_t *enp = eep->ee_enp;
+	uint32_t num_descs;
+	uint32_t label;
+	boolean_t should_abort;
+
+	EFX_EV_QSTAT_INCR(eep, EV_TX);
+
+	/* Discard events after RXQ/TXQ errors, or hardware not available */
+	if (enp->en_reset_flags &
+	    (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR | EFX_RESET_HW_UNAVAIL))
+		return (B_FALSE);
+
+	label = EFX_QWORD_FIELD(*eqp, ESF_GZ_EV_TXCMPL_Q_LABEL);
+
+	/*
+	 * On EF100 the EV_TX event reports the number of completed Tx
+	 * descriptors (on EF10, the event reports the low bits of the
+	 * index of the last completed descriptor).
+	 * The client driver completion callback will compute the
+	 * descriptor index, so that is not needed here.
+	 */
+	num_descs = EFX_QWORD_FIELD(*eqp, ESF_GZ_EV_TXCMPL_NUM_DESC);
+
+	EFSYS_PROBE2(tx_ndescs, uint32_t, label, unsigned int, num_descs);
+
+	EFSYS_ASSERT(eecp->eec_tx_ndescs != NULL);
+	should_abort = eecp->eec_tx_ndescs(arg, label, num_descs);
+
+	return (should_abort);
+}
+
 static	__checkReturn	boolean_t
 rhead_ev_mcdi(
 	__in		efx_evq_t *eep,
@@ -293,4 +620,68 @@ rhead_ev_mcdi(
 	return (ret);
 }
 
+#if EFSYS_OPT_DESC_PROXY
+static			boolean_t
+rhead_ev_ew_txq_desc(
+	__in		efx_evq_t *eep,
+	__in		efx_xword_t *eventp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	efx_oword_t txq_desc;
+	uint16_t vi_id;
+	boolean_t should_abort;
+
+	_NOTE(ARGUNUSED(eep))
+
+	vi_id = EFX_XWORD_FIELD(*eventp, ESF_GZ_EV_TXQ_DP_VI_ID);
+
+	/*
+	 * NOTE: This is the raw descriptor data, and has not been converted
+	 * to host endian. The handler callback must use the EFX_OWORD macros
+	 * to extract the descriptor fields as host endian values.
+	 */
+	txq_desc = eventp->ex_oword[0];
+
+	EFSYS_ASSERT(eecp->eec_desc_proxy_txq_desc != NULL);
+	should_abort = eecp->eec_desc_proxy_txq_desc(arg, vi_id, txq_desc);
+
+	return (should_abort);
+}
+#endif /* EFSYS_OPT_DESC_PROXY */
+
+
+#if EFSYS_OPT_DESC_PROXY
+static			boolean_t
+rhead_ev_ew_virtq_desc(
+	__in		efx_evq_t *eep,
+	__in		efx_xword_t *eventp,
+	__in		const efx_ev_callbacks_t *eecp,
+	__in_opt	void *arg)
+{
+	efx_oword_t virtq_desc;
+	uint16_t vi_id;
+	uint16_t avail;
+	boolean_t should_abort;
+
+	_NOTE(ARGUNUSED(eep))
+
+	vi_id = EFX_XWORD_FIELD(*eventp, ESF_GZ_EV_VQ_DP_VI_ID);
+	avail = EFX_XWORD_FIELD(*eventp, ESF_GZ_EV_VQ_DP_AVAIL_ENTRY);
+
+	/*
+	 * NOTE: This is the raw descriptor data, and has not been converted
+	 * to host endian. The handler callback must use the EFX_OWORD macros
+	 * to extract the descriptor fields as host endian values.
+	 */
+	virtq_desc = eventp->ex_oword[0];
+
+	EFSYS_ASSERT(eecp->eec_desc_proxy_virtq_desc != NULL);
+	should_abort =
+	    eecp->eec_desc_proxy_virtq_desc(arg, vi_id, avail, virtq_desc);
+
+	return (should_abort);
+}
+#endif /* EFSYS_OPT_DESC_PROXY */
+
 #endif	/* EFSYS_OPT_RIVERHEAD */