-static __rte_always_inline int
-run_default(struct rte_eth_dev *dev)
-{
- struct pmd_internals *p = dev->data->dev_private;
-
- /* Persistent context: Read Only (update not required) */
- struct rte_mbuf **pkts = p->soft.def.pkts;
- uint16_t nb_tx_queues = dev->data->nb_tx_queues;
-
- /* Persistent context: Read - Write (update required) */
- uint32_t txq_pos = p->soft.def.txq_pos;
- uint32_t pkts_len = p->soft.def.pkts_len;
- uint32_t flush_count = p->soft.def.flush_count;
-
- /* Not part of the persistent context */
- uint32_t pos;
- uint16_t i;
-
- /* Soft device TXQ read, Hard device TXQ write */
- for (i = 0; i < nb_tx_queues; i++) {
- struct rte_ring *txq = dev->data->tx_queues[txq_pos];
-
- /* Read soft device TXQ burst to packet enqueue buffer */
- pkts_len += rte_ring_sc_dequeue_burst(txq,
- (void **)&pkts[pkts_len],
- DEFAULT_BURST_SIZE,
- NULL);
-
- /* Increment soft device TXQ */
- txq_pos++;
- if (txq_pos >= nb_tx_queues)
- txq_pos = 0;
-
- /* Hard device TXQ write when complete burst is available */
- if (pkts_len >= DEFAULT_BURST_SIZE) {
- for (pos = 0; pos < pkts_len; )
- pos += rte_eth_tx_burst(p->hard.port_id,
- p->params.hard.tx_queue_id,
- &pkts[pos],
- (uint16_t)(pkts_len - pos));
-
- pkts_len = 0;
- flush_count = 0;
- break;
- }
- }
-
- if (flush_count >= FLUSH_COUNT_THRESHOLD) {
- for (pos = 0; pos < pkts_len; )
- pos += rte_eth_tx_burst(p->hard.port_id,
- p->params.hard.tx_queue_id,
- &pkts[pos],
- (uint16_t)(pkts_len - pos));
-
- pkts_len = 0;
- flush_count = 0;
- }
-
- p->soft.def.txq_pos = txq_pos;
- p->soft.def.pkts_len = pkts_len;
- p->soft.def.flush_count = flush_count + 1;
-
- return 0;
-}
-
-static __rte_always_inline int
-run_tm(struct rte_eth_dev *dev)
-{
- struct pmd_internals *p = dev->data->dev_private;
-
- /* Persistent context: Read Only (update not required) */
- struct rte_sched_port *sched = p->soft.tm.sched;
- struct rte_mbuf **pkts_enq = p->soft.tm.pkts_enq;
- struct rte_mbuf **pkts_deq = p->soft.tm.pkts_deq;
- uint32_t enq_bsz = p->params.soft.tm.enq_bsz;
- uint32_t deq_bsz = p->params.soft.tm.deq_bsz;
- uint16_t nb_tx_queues = dev->data->nb_tx_queues;
-
- /* Persistent context: Read - Write (update required) */
- uint32_t txq_pos = p->soft.tm.txq_pos;
- uint32_t pkts_enq_len = p->soft.tm.pkts_enq_len;
- uint32_t flush_count = p->soft.tm.flush_count;
-
- /* Not part of the persistent context */
- uint32_t pkts_deq_len, pos;
- uint16_t i;
-
- /* Soft device TXQ read, TM enqueue */
- for (i = 0; i < nb_tx_queues; i++) {
- struct rte_ring *txq = dev->data->tx_queues[txq_pos];
-
- /* Read TXQ burst to packet enqueue buffer */
- pkts_enq_len += rte_ring_sc_dequeue_burst(txq,
- (void **)&pkts_enq[pkts_enq_len],
- enq_bsz,
- NULL);
-
- /* Increment TXQ */
- txq_pos++;
- if (txq_pos >= nb_tx_queues)
- txq_pos = 0;
-
- /* TM enqueue when complete burst is available */
- if (pkts_enq_len >= enq_bsz) {
- rte_sched_port_enqueue(sched, pkts_enq, pkts_enq_len);
-
- pkts_enq_len = 0;
- flush_count = 0;
- break;
- }
- }
-
- if (flush_count >= FLUSH_COUNT_THRESHOLD) {
- if (pkts_enq_len)
- rte_sched_port_enqueue(sched, pkts_enq, pkts_enq_len);
-
- pkts_enq_len = 0;
- flush_count = 0;
- }
-
- p->soft.tm.txq_pos = txq_pos;
- p->soft.tm.pkts_enq_len = pkts_enq_len;
- p->soft.tm.flush_count = flush_count + 1;
-
- /* TM dequeue, Hard device TXQ write */
- pkts_deq_len = rte_sched_port_dequeue(sched, pkts_deq, deq_bsz);
-
- for (pos = 0; pos < pkts_deq_len; )
- pos += rte_eth_tx_burst(p->hard.port_id,
- p->params.hard.tx_queue_id,
- &pkts_deq[pos],
- (uint16_t)(pkts_deq_len - pos));
-
- return 0;
-}
-