X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fevent%2Focteontx2%2Fotx2_worker.h;h=4e971f27c7f6c5d77436b346ff47efd70983a22d;hb=e9a356e2fc719f87936d8e30058ac137f4b8c5c4;hp=f06ff064ed11419adc4458ae3ff7ab921255511f;hpb=4095660db72f4375c4a7009f7d298310737fef8f;p=dpdk.git

diff --git a/drivers/event/octeontx2/otx2_worker.h b/drivers/event/octeontx2/otx2_worker.h
index f06ff064ed..4e971f27c7 100644
--- a/drivers/event/octeontx2/otx2_worker.h
+++ b/drivers/event/octeontx2/otx2_worker.h
@@ -14,15 +14,20 @@
 /* SSO Operations */
 
 static __rte_always_inline uint16_t
-otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev)
+otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev,
+		     const uint32_t flags, const void * const lookup_mem)
 {
 	union otx2_sso_event event;
+	uint64_t tstamp_ptr;
 	uint64_t get_work1;
+	uint64_t mbuf;
 
 	otx2_write64(BIT_ULL(16) | /* wait for work. */
 		     1, /* Use Mask set 0. */
 		     ws->getwrk_op);
 
+	if (flags & NIX_RX_OFFLOAD_PTYPE_F)
+		rte_prefetch_non_temporal(lookup_mem);
 #ifdef RTE_ARCH_ARM64
 	asm volatile(
 			"		ldr %[tag], [%[tag_loc]]	\n"
@@ -34,9 +39,12 @@ otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev)
 			"		ldr %[wqp], [%[wqp_loc]]	\n"
 			"		tbnz %[tag], 63, rty%=		\n"
 			"done%=:	dmb ld				\n"
-			"		prfm pldl1keep, [%[wqp]]	\n"
+			"		prfm pldl1keep, [%[wqp], #8]	\n"
+			"		sub %[mbuf], %[wqp], #0x80	\n"
+			"		prfm pldl1keep, [%[mbuf]]	\n"
 			: [tag] "=&r" (event.get_work0),
-			  [wqp] "=&r" (get_work1)
+			  [wqp] "=&r" (get_work1),
+			  [mbuf] "=&r" (mbuf)
 			: [tag_loc] "r" (ws->tag_op),
 			  [wqp_loc] "r" (ws->wqp_op)
 			);
@@ -47,6 +55,8 @@ otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev)
 
 	get_work1 = otx2_read64(ws->wqp_op);
 	rte_prefetch0((const void *)get_work1);
+	mbuf = (uint64_t)((char *)get_work1 - sizeof(struct rte_mbuf));
+	rte_prefetch0((const void *)mbuf);
 #endif
 
 	event.get_work0 = (event.get_work0 & (0x3ull << 32)) << 6 |
@@ -55,6 +65,17 @@ otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev)
 	ws->cur_tt = event.sched_type;
 	ws->cur_grp = event.queue_id;
 
+	if (event.sched_type != SSO_TT_EMPTY &&
+	    event.event_type == RTE_EVENT_TYPE_ETHDEV) {
+		otx2_wqe_to_mbuf(get_work1, mbuf, event.sub_event_type,
+				 (uint32_t) event.get_work0, flags, lookup_mem);
+		/* Extracting tstamp, if PTP enabled*/
+		tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *)get_work1)
+					     + OTX2_SSO_WQE_SG_PTR);
+		otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp,
+					flags, (uint64_t *)tstamp_ptr);
+		get_work1 = mbuf;
+	}
 
 	ev->event = event.get_work0;
 	ev->u64 = get_work1;
@@ -64,10 +85,13 @@ otx2_ssogws_get_work(struct otx2_ssogws *ws, struct rte_event *ev)
 
 /* Used in cleaning up workslot. */
 static __rte_always_inline uint16_t
-otx2_ssogws_get_work_empty(struct otx2_ssogws *ws, struct rte_event *ev)
+otx2_ssogws_get_work_empty(struct otx2_ssogws *ws, struct rte_event *ev,
+			   const uint32_t flags)
 {
 	union otx2_sso_event event;
+	uint64_t tstamp_ptr;
 	uint64_t get_work1;
+	uint64_t mbuf;
 
 #ifdef RTE_ARCH_ARM64
 	asm volatile(
@@ -80,9 +104,12 @@ otx2_ssogws_get_work_empty(struct otx2_ssogws *ws, struct rte_event *ev)
 			"		ldr %[wqp], [%[wqp_loc]]	\n"
 			"		tbnz %[tag], 63, rty%=		\n"
 			"done%=:	dmb ld				\n"
-			"		prfm pldl1keep, [%[wqp]]	\n"
+			"		prfm pldl1keep, [%[wqp], #8]	\n"
+			"		sub %[mbuf], %[wqp], #0x80	\n"
+			"		prfm pldl1keep, [%[mbuf]]	\n"
 			: [tag] "=&r" (event.get_work0),
-			  [wqp] "=&r" (get_work1)
+			  [wqp] "=&r" (get_work1),
+			  [mbuf] "=&r" (mbuf)
 			: [tag_loc] "r" (ws->tag_op),
 			  [wqp_loc] "r" (ws->wqp_op)
 			);
@@ -92,7 +119,9 @@ otx2_ssogws_get_work_empty(struct otx2_ssogws *ws, struct rte_event *ev)
 		event.get_work0 = otx2_read64(ws->tag_op);
 
 	get_work1 = otx2_read64(ws->wqp_op);
-	rte_prefetch0((const void *)get_work1);
+	rte_prefetch_non_temporal((const void *)get_work1);
+	mbuf = (uint64_t)((char *)get_work1 - sizeof(struct rte_mbuf));
+	rte_prefetch_non_temporal((const void *)mbuf);
 #endif
 
 	event.get_work0 = (event.get_work0 & (0x3ull << 32)) << 6 |
@@ -101,6 +130,18 @@ otx2_ssogws_get_work_empty(struct otx2_ssogws *ws, struct rte_event *ev)
 	ws->cur_tt = event.sched_type;
 	ws->cur_grp = event.queue_id;
 
+	if (event.sched_type != SSO_TT_EMPTY &&
+	    event.event_type == RTE_EVENT_TYPE_ETHDEV) {
+		otx2_wqe_to_mbuf(get_work1, mbuf, event.sub_event_type,
+				 (uint32_t) event.get_work0, flags, NULL);
+		/* Extracting tstamp, if PTP enabled*/
+		tstamp_ptr = *(uint64_t *)(((struct nix_wqe_hdr_s *)get_work1)
+					     + OTX2_SSO_WQE_SG_PTR);
+		otx2_nix_mbuf_to_tstamp((struct rte_mbuf *)mbuf, ws->tstamp,
+					flags, (uint64_t *)tstamp_ptr);
+		get_work1 = mbuf;
+	}
+
 	ev->event = event.get_work0;
 	ev->u64 = get_work1;
 
@@ -184,4 +225,55 @@ otx2_ssogws_swtag_wait(struct otx2_ssogws *ws)
 #endif
 }
 
+static __rte_always_inline void
+otx2_ssogws_head_wait(struct otx2_ssogws *ws, const uint8_t wait_flag)
+{
+	while (wait_flag && !(otx2_read64(ws->tag_op) & BIT_ULL(35)))
+		;
+
+	rte_cio_wmb();
+}
+
+static __rte_always_inline const struct otx2_eth_txq *
+otx2_ssogws_xtract_meta(struct rte_mbuf *m)
+{
+	return rte_eth_devices[m->port].data->tx_queues[
+			rte_event_eth_tx_adapter_txq_get(m)];
+}
+
+static __rte_always_inline void
+otx2_ssogws_prepare_pkt(const struct otx2_eth_txq *txq, struct rte_mbuf *m,
+			uint64_t *cmd, const uint32_t flags)
+{
+	otx2_lmt_mov(cmd, txq->cmd, otx2_nix_tx_ext_subs(flags));
+	otx2_nix_xmit_prepare(m, cmd, flags);
+}
+
+static __rte_always_inline uint16_t
+otx2_ssogws_event_tx(struct otx2_ssogws *ws, struct rte_event ev[],
+		     uint64_t *cmd, const uint32_t flags)
+{
+	struct rte_mbuf *m = ev[0].mbuf;
+	const struct otx2_eth_txq *txq = otx2_ssogws_xtract_meta(m);
+
+	/* Perform header writes before barrier for TSO */
+	otx2_nix_xmit_prepare_tso(m, flags);
+	otx2_ssogws_head_wait(ws, !ev->sched_type);
+	otx2_ssogws_prepare_pkt(txq, m, cmd, flags);
+
+	if (flags & NIX_TX_MULTI_SEG_F) {
+		const uint16_t segdw = otx2_nix_prepare_mseg(m, cmd, flags);
+		otx2_nix_xmit_prepare_tstamp(cmd, &txq->cmd[0],
+					     m->ol_flags, segdw, flags);
+		otx2_nix_xmit_mseg_one(cmd, txq->lmt_addr, txq->io_addr, segdw);
+	} else {
+		/* Passing no of segdw as 4: HDR + EXT + SG + SMEM */
+		otx2_nix_xmit_prepare_tstamp(cmd, &txq->cmd[0],
+					     m->ol_flags, 4, flags);
+		otx2_nix_xmit_one(cmd, txq->lmt_addr, txq->io_addr, flags);
+	}
+
+	return 1;
+}
+
 #endif