84f348fa0512c7e8ce1b4d7ec157fe4ca3d9fbf0
[dpdk.git] / drivers / net / vmxnet3 / vmxnet3_rxtx.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2015 Intel Corporation
3  */
4
5 #include <sys/queue.h>
6
7 #include <stdio.h>
8 #include <stdlib.h>
9 #include <string.h>
10 #include <errno.h>
11 #include <stdint.h>
12 #include <stdarg.h>
13 #include <unistd.h>
14 #include <inttypes.h>
15
16 #include <rte_byteorder.h>
17 #include <rte_common.h>
18 #include <rte_cycles.h>
19 #include <rte_log.h>
20 #include <rte_debug.h>
21 #include <rte_interrupts.h>
22 #include <rte_pci.h>
23 #include <rte_memory.h>
24 #include <rte_memzone.h>
25 #include <rte_launch.h>
26 #include <rte_eal.h>
27 #include <rte_per_lcore.h>
28 #include <rte_lcore.h>
29 #include <rte_atomic.h>
30 #include <rte_branch_prediction.h>
31 #include <rte_mempool.h>
32 #include <rte_malloc.h>
33 #include <rte_mbuf.h>
34 #include <rte_ether.h>
35 #include <rte_ethdev_driver.h>
36 #include <rte_prefetch.h>
37 #include <rte_ip.h>
38 #include <rte_udp.h>
39 #include <rte_tcp.h>
40 #include <rte_sctp.h>
41 #include <rte_string_fns.h>
42 #include <rte_errno.h>
43 #include <rte_net.h>
44
45 #include "base/vmxnet3_defs.h"
46 #include "vmxnet3_ring.h"
47
48 #include "vmxnet3_logs.h"
49 #include "vmxnet3_ethdev.h"
50
51 #define VMXNET3_TX_OFFLOAD_MASK ( \
52                 PKT_TX_VLAN_PKT | \
53                 PKT_TX_L4_MASK |  \
54                 PKT_TX_TCP_SEG)
55
56 #define VMXNET3_TX_OFFLOAD_NOTSUP_MASK  \
57         (PKT_TX_OFFLOAD_MASK ^ VMXNET3_TX_OFFLOAD_MASK)
58
59 static const uint32_t rxprod_reg[2] = {VMXNET3_REG_RXPROD, VMXNET3_REG_RXPROD2};
60
61 static int vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t*, uint8_t);
62 static void vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *);
63 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
64 static void vmxnet3_rxq_dump(struct vmxnet3_rx_queue *);
65 static void vmxnet3_txq_dump(struct vmxnet3_tx_queue *);
66 #endif
67
68 #ifdef RTE_LIBRTE_VMXNET3_DEBUG_DRIVER_NOT_USED
69 static void
70 vmxnet3_rxq_dump(struct vmxnet3_rx_queue *rxq)
71 {
72         uint32_t avail = 0;
73
74         if (rxq == NULL)
75                 return;
76
77         PMD_RX_LOG(DEBUG,
78                    "RXQ: cmd0 base : %p cmd1 base : %p comp ring base : %p.",
79                    rxq->cmd_ring[0].base, rxq->cmd_ring[1].base, rxq->comp_ring.base);
80         PMD_RX_LOG(DEBUG,
81                    "RXQ: cmd0 basePA : 0x%lx cmd1 basePA : 0x%lx comp ring basePA : 0x%lx.",
82                    (unsigned long)rxq->cmd_ring[0].basePA,
83                    (unsigned long)rxq->cmd_ring[1].basePA,
84                    (unsigned long)rxq->comp_ring.basePA);
85
86         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[0]);
87         PMD_RX_LOG(DEBUG,
88                    "RXQ:cmd0: size=%u; free=%u; next2proc=%u; queued=%u",
89                    (uint32_t)rxq->cmd_ring[0].size, avail,
90                    rxq->comp_ring.next2proc,
91                    rxq->cmd_ring[0].size - avail);
92
93         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[1]);
94         PMD_RX_LOG(DEBUG, "RXQ:cmd1 size=%u; free=%u; next2proc=%u; queued=%u",
95                    (uint32_t)rxq->cmd_ring[1].size, avail, rxq->comp_ring.next2proc,
96                    rxq->cmd_ring[1].size - avail);
97
98 }
99
100 static void
101 vmxnet3_txq_dump(struct vmxnet3_tx_queue *txq)
102 {
103         uint32_t avail = 0;
104
105         if (txq == NULL)
106                 return;
107
108         PMD_TX_LOG(DEBUG, "TXQ: cmd base : %p comp ring base : %p data ring base : %p.",
109                    txq->cmd_ring.base, txq->comp_ring.base, txq->data_ring.base);
110         PMD_TX_LOG(DEBUG, "TXQ: cmd basePA : 0x%lx comp ring basePA : 0x%lx data ring basePA : 0x%lx.",
111                    (unsigned long)txq->cmd_ring.basePA,
112                    (unsigned long)txq->comp_ring.basePA,
113                    (unsigned long)txq->data_ring.basePA);
114
115         avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
116         PMD_TX_LOG(DEBUG, "TXQ: size=%u; free=%u; next2proc=%u; queued=%u",
117                    (uint32_t)txq->cmd_ring.size, avail,
118                    txq->comp_ring.next2proc, txq->cmd_ring.size - avail);
119 }
120 #endif
121
122 static void
123 vmxnet3_tx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
124 {
125         while (ring->next2comp != ring->next2fill) {
126                 /* No need to worry about desc ownership, device is quiesced by now. */
127                 vmxnet3_buf_info_t *buf_info = ring->buf_info + ring->next2comp;
128
129                 if (buf_info->m) {
130                         rte_pktmbuf_free(buf_info->m);
131                         buf_info->m = NULL;
132                         buf_info->bufPA = 0;
133                         buf_info->len = 0;
134                 }
135                 vmxnet3_cmd_ring_adv_next2comp(ring);
136         }
137 }
138
139 static void
140 vmxnet3_rx_cmd_ring_release_mbufs(vmxnet3_cmd_ring_t *ring)
141 {
142         uint32_t i;
143
144         for (i = 0; i < ring->size; i++) {
145                 /* No need to worry about desc ownership, device is quiesced by now. */
146                 vmxnet3_buf_info_t *buf_info = &ring->buf_info[i];
147
148                 if (buf_info->m) {
149                         rte_pktmbuf_free_seg(buf_info->m);
150                         buf_info->m = NULL;
151                         buf_info->bufPA = 0;
152                         buf_info->len = 0;
153                 }
154                 vmxnet3_cmd_ring_adv_next2comp(ring);
155         }
156 }
157
158 static void
159 vmxnet3_cmd_ring_release(vmxnet3_cmd_ring_t *ring)
160 {
161         rte_free(ring->buf_info);
162         ring->buf_info = NULL;
163 }
164
165 void
166 vmxnet3_dev_tx_queue_release(void *txq)
167 {
168         vmxnet3_tx_queue_t *tq = txq;
169
170         if (tq != NULL) {
171                 /* Release mbufs */
172                 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
173                 /* Release the cmd_ring */
174                 vmxnet3_cmd_ring_release(&tq->cmd_ring);
175                 /* Release the memzone */
176                 rte_memzone_free(tq->mz);
177                 /* Release the queue */
178                 rte_free(tq);
179         }
180 }
181
182 void
183 vmxnet3_dev_rx_queue_release(void *rxq)
184 {
185         int i;
186         vmxnet3_rx_queue_t *rq = rxq;
187
188         if (rq != NULL) {
189                 /* Release mbufs */
190                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
191                         vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
192
193                 /* Release both the cmd_rings */
194                 for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
195                         vmxnet3_cmd_ring_release(&rq->cmd_ring[i]);
196
197                 /* Release the memzone */
198                 rte_memzone_free(rq->mz);
199
200                 /* Release the queue */
201                 rte_free(rq);
202         }
203 }
204
205 static void
206 vmxnet3_dev_tx_queue_reset(void *txq)
207 {
208         vmxnet3_tx_queue_t *tq = txq;
209         struct vmxnet3_cmd_ring *ring = &tq->cmd_ring;
210         struct vmxnet3_comp_ring *comp_ring = &tq->comp_ring;
211         struct vmxnet3_data_ring *data_ring = &tq->data_ring;
212         int size;
213
214         if (tq != NULL) {
215                 /* Release the cmd_ring mbufs */
216                 vmxnet3_tx_cmd_ring_release_mbufs(&tq->cmd_ring);
217         }
218
219         /* Tx vmxnet rings structure initialization*/
220         ring->next2fill = 0;
221         ring->next2comp = 0;
222         ring->gen = VMXNET3_INIT_GEN;
223         comp_ring->next2proc = 0;
224         comp_ring->gen = VMXNET3_INIT_GEN;
225
226         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
227         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
228         size += tq->txdata_desc_size * data_ring->size;
229
230         memset(ring->base, 0, size);
231 }
232
233 static void
234 vmxnet3_dev_rx_queue_reset(void *rxq)
235 {
236         int i;
237         vmxnet3_rx_queue_t *rq = rxq;
238         struct vmxnet3_hw *hw = rq->hw;
239         struct vmxnet3_cmd_ring *ring0, *ring1;
240         struct vmxnet3_comp_ring *comp_ring;
241         struct vmxnet3_rx_data_ring *data_ring = &rq->data_ring;
242         int size;
243
244         /* Release both the cmd_rings mbufs */
245         for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++)
246                 vmxnet3_rx_cmd_ring_release_mbufs(&rq->cmd_ring[i]);
247
248         ring0 = &rq->cmd_ring[0];
249         ring1 = &rq->cmd_ring[1];
250         comp_ring = &rq->comp_ring;
251
252         /* Rx vmxnet rings structure initialization */
253         ring0->next2fill = 0;
254         ring1->next2fill = 0;
255         ring0->next2comp = 0;
256         ring1->next2comp = 0;
257         ring0->gen = VMXNET3_INIT_GEN;
258         ring1->gen = VMXNET3_INIT_GEN;
259         comp_ring->next2proc = 0;
260         comp_ring->gen = VMXNET3_INIT_GEN;
261
262         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
263         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
264         if (VMXNET3_VERSION_GE_3(hw) && rq->data_desc_size)
265                 size += rq->data_desc_size * data_ring->size;
266
267         memset(ring0->base, 0, size);
268 }
269
270 void
271 vmxnet3_dev_clear_queues(struct rte_eth_dev *dev)
272 {
273         unsigned i;
274
275         PMD_INIT_FUNC_TRACE();
276
277         for (i = 0; i < dev->data->nb_tx_queues; i++) {
278                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
279
280                 if (txq != NULL) {
281                         txq->stopped = TRUE;
282                         vmxnet3_dev_tx_queue_reset(txq);
283                 }
284         }
285
286         for (i = 0; i < dev->data->nb_rx_queues; i++) {
287                 struct vmxnet3_rx_queue *rxq = dev->data->rx_queues[i];
288
289                 if (rxq != NULL) {
290                         rxq->stopped = TRUE;
291                         vmxnet3_dev_rx_queue_reset(rxq);
292                 }
293         }
294 }
295
296 static int
297 vmxnet3_unmap_pkt(uint16_t eop_idx, vmxnet3_tx_queue_t *txq)
298 {
299         int completed = 0;
300         struct rte_mbuf *mbuf;
301
302         /* Release cmd_ring descriptor and free mbuf */
303         RTE_ASSERT(txq->cmd_ring.base[eop_idx].txd.eop == 1);
304
305         mbuf = txq->cmd_ring.buf_info[eop_idx].m;
306         if (mbuf == NULL)
307                 rte_panic("EOP desc does not point to a valid mbuf");
308         rte_pktmbuf_free(mbuf);
309
310         txq->cmd_ring.buf_info[eop_idx].m = NULL;
311
312         while (txq->cmd_ring.next2comp != eop_idx) {
313                 /* no out-of-order completion */
314                 RTE_ASSERT(txq->cmd_ring.base[txq->cmd_ring.next2comp].txd.cq == 0);
315                 vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
316                 completed++;
317         }
318
319         /* Mark the txd for which tcd was generated as completed */
320         vmxnet3_cmd_ring_adv_next2comp(&txq->cmd_ring);
321
322         return completed + 1;
323 }
324
325 static void
326 vmxnet3_tq_tx_complete(vmxnet3_tx_queue_t *txq)
327 {
328         int completed = 0;
329         vmxnet3_comp_ring_t *comp_ring = &txq->comp_ring;
330         struct Vmxnet3_TxCompDesc *tcd = (struct Vmxnet3_TxCompDesc *)
331                 (comp_ring->base + comp_ring->next2proc);
332
333         while (tcd->gen == comp_ring->gen) {
334                 completed += vmxnet3_unmap_pkt(tcd->txdIdx, txq);
335
336                 vmxnet3_comp_ring_adv_next2proc(comp_ring);
337                 tcd = (struct Vmxnet3_TxCompDesc *)(comp_ring->base +
338                                                     comp_ring->next2proc);
339         }
340
341         PMD_TX_LOG(DEBUG, "Processed %d tx comps & command descs.", completed);
342 }
343
344 uint16_t
345 vmxnet3_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
346         uint16_t nb_pkts)
347 {
348         int32_t ret;
349         uint32_t i;
350         uint64_t ol_flags;
351         struct rte_mbuf *m;
352
353         for (i = 0; i != nb_pkts; i++) {
354                 m = tx_pkts[i];
355                 ol_flags = m->ol_flags;
356
357                 /* Non-TSO packet cannot occupy more than
358                  * VMXNET3_MAX_TXD_PER_PKT TX descriptors.
359                  */
360                 if ((ol_flags & PKT_TX_TCP_SEG) == 0 &&
361                                 m->nb_segs > VMXNET3_MAX_TXD_PER_PKT) {
362                         rte_errno = -EINVAL;
363                         return i;
364                 }
365
366                 /* check that only supported TX offloads are requested. */
367                 if ((ol_flags & VMXNET3_TX_OFFLOAD_NOTSUP_MASK) != 0 ||
368                                 (ol_flags & PKT_TX_L4_MASK) ==
369                                 PKT_TX_SCTP_CKSUM) {
370                         rte_errno = -ENOTSUP;
371                         return i;
372                 }
373
374 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
375                 ret = rte_validate_tx_offload(m);
376                 if (ret != 0) {
377                         rte_errno = ret;
378                         return i;
379                 }
380 #endif
381                 ret = rte_net_intel_cksum_prepare(m);
382                 if (ret != 0) {
383                         rte_errno = ret;
384                         return i;
385                 }
386         }
387
388         return i;
389 }
390
391 uint16_t
392 vmxnet3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
393                   uint16_t nb_pkts)
394 {
395         uint16_t nb_tx;
396         vmxnet3_tx_queue_t *txq = tx_queue;
397         struct vmxnet3_hw *hw = txq->hw;
398         Vmxnet3_TxQueueCtrl *txq_ctrl = &txq->shared->ctrl;
399         uint32_t deferred = rte_le_to_cpu_32(txq_ctrl->txNumDeferred);
400
401         if (unlikely(txq->stopped)) {
402                 PMD_TX_LOG(DEBUG, "Tx queue is stopped.");
403                 return 0;
404         }
405
406         /* Free up the comp_descriptors aggressively */
407         vmxnet3_tq_tx_complete(txq);
408
409         nb_tx = 0;
410         while (nb_tx < nb_pkts) {
411                 Vmxnet3_GenericDesc *gdesc;
412                 vmxnet3_buf_info_t *tbi;
413                 uint32_t first2fill, avail, dw2;
414                 struct rte_mbuf *txm = tx_pkts[nb_tx];
415                 struct rte_mbuf *m_seg = txm;
416                 int copy_size = 0;
417                 bool tso = (txm->ol_flags & PKT_TX_TCP_SEG) != 0;
418                 /* # of descriptors needed for a packet. */
419                 unsigned count = txm->nb_segs;
420
421                 avail = vmxnet3_cmd_ring_desc_avail(&txq->cmd_ring);
422                 if (count > avail) {
423                         /* Is command ring full? */
424                         if (unlikely(avail == 0)) {
425                                 PMD_TX_LOG(DEBUG, "No free ring descriptors");
426                                 txq->stats.tx_ring_full++;
427                                 txq->stats.drop_total += (nb_pkts - nb_tx);
428                                 break;
429                         }
430
431                         /* Command ring is not full but cannot handle the
432                          * multi-segmented packet. Let's try the next packet
433                          * in this case.
434                          */
435                         PMD_TX_LOG(DEBUG, "Running out of ring descriptors "
436                                    "(avail %d needed %d)", avail, count);
437                         txq->stats.drop_total++;
438                         if (tso)
439                                 txq->stats.drop_tso++;
440                         rte_pktmbuf_free(txm);
441                         nb_tx++;
442                         continue;
443                 }
444
445                 /* Drop non-TSO packet that is excessively fragmented */
446                 if (unlikely(!tso && count > VMXNET3_MAX_TXD_PER_PKT)) {
447                         PMD_TX_LOG(ERR, "Non-TSO packet cannot occupy more than %d tx "
448                                    "descriptors. Packet dropped.", VMXNET3_MAX_TXD_PER_PKT);
449                         txq->stats.drop_too_many_segs++;
450                         txq->stats.drop_total++;
451                         rte_pktmbuf_free(txm);
452                         nb_tx++;
453                         continue;
454                 }
455
456                 if (txm->nb_segs == 1 &&
457                     rte_pktmbuf_pkt_len(txm) <= txq->txdata_desc_size) {
458                         struct Vmxnet3_TxDataDesc *tdd;
459
460                         tdd = (struct Vmxnet3_TxDataDesc *)
461                                 ((uint8 *)txq->data_ring.base +
462                                  txq->cmd_ring.next2fill *
463                                  txq->txdata_desc_size);
464                         copy_size = rte_pktmbuf_pkt_len(txm);
465                         rte_memcpy(tdd->data, rte_pktmbuf_mtod(txm, char *), copy_size);
466                 }
467
468                 /* use the previous gen bit for the SOP desc */
469                 dw2 = (txq->cmd_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
470                 first2fill = txq->cmd_ring.next2fill;
471                 do {
472                         /* Remember the transmit buffer for cleanup */
473                         tbi = txq->cmd_ring.buf_info + txq->cmd_ring.next2fill;
474
475                         /* NB: the following assumes that VMXNET3 maximum
476                          * transmit buffer size (16K) is greater than
477                          * maximum size of mbuf segment size.
478                          */
479                         gdesc = txq->cmd_ring.base + txq->cmd_ring.next2fill;
480                         if (copy_size) {
481                                 uint64 offset =
482                                         (uint64)txq->cmd_ring.next2fill *
483                                                         txq->txdata_desc_size;
484                                 gdesc->txd.addr =
485                                         rte_cpu_to_le_64(txq->data_ring.basePA +
486                                                          offset);
487                         } else {
488                                 gdesc->txd.addr = rte_mbuf_data_iova(m_seg);
489                         }
490
491                         gdesc->dword[2] = dw2 | m_seg->data_len;
492                         gdesc->dword[3] = 0;
493
494                         /* move to the next2fill descriptor */
495                         vmxnet3_cmd_ring_adv_next2fill(&txq->cmd_ring);
496
497                         /* use the right gen for non-SOP desc */
498                         dw2 = txq->cmd_ring.gen << VMXNET3_TXD_GEN_SHIFT;
499                 } while ((m_seg = m_seg->next) != NULL);
500
501                 /* set the last buf_info for the pkt */
502                 tbi->m = txm;
503                 /* Update the EOP descriptor */
504                 gdesc->dword[3] |= VMXNET3_TXD_EOP | VMXNET3_TXD_CQ;
505
506                 /* Add VLAN tag if present */
507                 gdesc = txq->cmd_ring.base + first2fill;
508                 if (txm->ol_flags & PKT_TX_VLAN_PKT) {
509                         gdesc->txd.ti = 1;
510                         gdesc->txd.tci = txm->vlan_tci;
511                 }
512
513                 if (tso) {
514                         uint16_t mss = txm->tso_segsz;
515
516                         RTE_ASSERT(mss > 0);
517
518                         gdesc->txd.hlen = txm->l2_len + txm->l3_len + txm->l4_len;
519                         gdesc->txd.om = VMXNET3_OM_TSO;
520                         gdesc->txd.msscof = mss;
521
522                         deferred += (rte_pktmbuf_pkt_len(txm) - gdesc->txd.hlen + mss - 1) / mss;
523                 } else if (txm->ol_flags & PKT_TX_L4_MASK) {
524                         gdesc->txd.om = VMXNET3_OM_CSUM;
525                         gdesc->txd.hlen = txm->l2_len + txm->l3_len;
526
527                         switch (txm->ol_flags & PKT_TX_L4_MASK) {
528                         case PKT_TX_TCP_CKSUM:
529                                 gdesc->txd.msscof = gdesc->txd.hlen + offsetof(struct tcp_hdr, cksum);
530                                 break;
531                         case PKT_TX_UDP_CKSUM:
532                                 gdesc->txd.msscof = gdesc->txd.hlen + offsetof(struct udp_hdr, dgram_cksum);
533                                 break;
534                         default:
535                                 PMD_TX_LOG(WARNING, "requested cksum offload not supported %#llx",
536                                            txm->ol_flags & PKT_TX_L4_MASK);
537                                 abort();
538                         }
539                         deferred++;
540                 } else {
541                         gdesc->txd.hlen = 0;
542                         gdesc->txd.om = VMXNET3_OM_NONE;
543                         gdesc->txd.msscof = 0;
544                         deferred++;
545                 }
546
547                 /* flip the GEN bit on the SOP */
548                 rte_compiler_barrier();
549                 gdesc->dword[2] ^= VMXNET3_TXD_GEN;
550
551                 txq_ctrl->txNumDeferred = rte_cpu_to_le_32(deferred);
552                 nb_tx++;
553         }
554
555         PMD_TX_LOG(DEBUG, "vmxnet3 txThreshold: %u", rte_le_to_cpu_32(txq_ctrl->txThreshold));
556
557         if (deferred >= rte_le_to_cpu_32(txq_ctrl->txThreshold)) {
558                 txq_ctrl->txNumDeferred = 0;
559                 /* Notify vSwitch that packets are available. */
560                 VMXNET3_WRITE_BAR0_REG(hw, (VMXNET3_REG_TXPROD + txq->queue_id * VMXNET3_REG_ALIGN),
561                                        txq->cmd_ring.next2fill);
562         }
563
564         return nb_tx;
565 }
566
567 static inline void
568 vmxnet3_renew_desc(vmxnet3_rx_queue_t *rxq, uint8_t ring_id,
569                    struct rte_mbuf *mbuf)
570 {
571         uint32_t val;
572         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
573         struct Vmxnet3_RxDesc *rxd =
574                 (struct Vmxnet3_RxDesc *)(ring->base + ring->next2fill);
575         vmxnet3_buf_info_t *buf_info = &ring->buf_info[ring->next2fill];
576
577         if (ring_id == 0) {
578                 /* Usually: One HEAD type buf per packet
579                  * val = (ring->next2fill % rxq->hw->bufs_per_pkt) ?
580                  * VMXNET3_RXD_BTYPE_BODY : VMXNET3_RXD_BTYPE_HEAD;
581                  */
582
583                 /* We use single packet buffer so all heads here */
584                 val = VMXNET3_RXD_BTYPE_HEAD;
585         } else {
586                 /* All BODY type buffers for 2nd ring */
587                 val = VMXNET3_RXD_BTYPE_BODY;
588         }
589
590         /*
591          * Load mbuf pointer into buf_info[ring_size]
592          * buf_info structure is equivalent to cookie for virtio-virtqueue
593          */
594         buf_info->m = mbuf;
595         buf_info->len = (uint16_t)(mbuf->buf_len - RTE_PKTMBUF_HEADROOM);
596         buf_info->bufPA = rte_mbuf_data_iova_default(mbuf);
597
598         /* Load Rx Descriptor with the buffer's GPA */
599         rxd->addr = buf_info->bufPA;
600
601         /* After this point rxd->addr MUST not be NULL */
602         rxd->btype = val;
603         rxd->len = buf_info->len;
604         /* Flip gen bit at the end to change ownership */
605         rxd->gen = ring->gen;
606
607         vmxnet3_cmd_ring_adv_next2fill(ring);
608 }
609 /*
610  *  Allocates mbufs and clusters. Post rx descriptors with buffer details
611  *  so that device can receive packets in those buffers.
612  *  Ring layout:
613  *      Among the two rings, 1st ring contains buffers of type 0 and type 1.
614  *      bufs_per_pkt is set such that for non-LRO cases all the buffers required
615  *      by a frame will fit in 1st ring (1st buf of type0 and rest of type1).
616  *      2nd ring contains buffers of type 1 alone. Second ring mostly be used
617  *      only for LRO.
618  */
619 static int
620 vmxnet3_post_rx_bufs(vmxnet3_rx_queue_t *rxq, uint8_t ring_id)
621 {
622         int err = 0;
623         uint32_t i = 0;
624         struct vmxnet3_cmd_ring *ring = &rxq->cmd_ring[ring_id];
625
626         while (vmxnet3_cmd_ring_desc_avail(ring) > 0) {
627                 struct rte_mbuf *mbuf;
628
629                 /* Allocate blank mbuf for the current Rx Descriptor */
630                 mbuf = rte_mbuf_raw_alloc(rxq->mp);
631                 if (unlikely(mbuf == NULL)) {
632                         PMD_RX_LOG(ERR, "Error allocating mbuf");
633                         rxq->stats.rx_buf_alloc_failure++;
634                         err = ENOMEM;
635                         break;
636                 }
637
638                 vmxnet3_renew_desc(rxq, ring_id, mbuf);
639                 i++;
640         }
641
642         /* Return error only if no buffers are posted at present */
643         if (vmxnet3_cmd_ring_desc_avail(ring) >= (ring->size - 1))
644                 return -err;
645         else
646                 return i;
647 }
648
649
650 /* Receive side checksum and other offloads */
651 static inline void
652 vmxnet3_rx_offload(struct vmxnet3_hw *hw, const Vmxnet3_RxCompDesc *rcd,
653                 struct rte_mbuf *rxm, const uint8_t sop)
654 {
655         (void)hw;
656
657         /* Offloads set in sop */
658         if (sop) {
659                 /* Check for RSS */
660                 if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE) {
661                         rxm->ol_flags |= PKT_RX_RSS_HASH;
662                         rxm->hash.rss = rcd->rssHash;
663                 }
664
665                 /* Check packet type, checksum errors. Only IPv4 for now. */
666                 if (rcd->v4) {
667                         rxm->packet_type = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
668
669                         if (!rcd->cnc) {
670                                 if (!rcd->ipc)
671                                         rxm->ol_flags |= PKT_RX_IP_CKSUM_BAD;
672
673                                 if ((rcd->tcp || rcd->udp) && !rcd->tuc)
674                                         rxm->ol_flags |= PKT_RX_L4_CKSUM_BAD;
675                         }
676                 } else {
677                         rxm->packet_type = RTE_PTYPE_UNKNOWN;
678                 }
679         } else { /* Offloads set in eop */
680                 /* Check for hardware stripped VLAN tag */
681                 if (rcd->ts) {
682                         rxm->ol_flags |= (PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED);
683                         rxm->vlan_tci = rte_le_to_cpu_16((uint16_t)rcd->tci);
684                 }
685         }
686 }
687
688 /*
689  * Process the Rx Completion Ring of given vmxnet3_rx_queue
690  * for nb_pkts burst and return the number of packets received
691  */
692 uint16_t
693 vmxnet3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
694 {
695         uint16_t nb_rx;
696         uint32_t nb_rxd, idx;
697         uint8_t ring_idx;
698         vmxnet3_rx_queue_t *rxq;
699         Vmxnet3_RxCompDesc *rcd;
700         vmxnet3_buf_info_t *rbi;
701         Vmxnet3_RxDesc *rxd;
702         struct rte_mbuf *rxm = NULL;
703         struct vmxnet3_hw *hw;
704
705         nb_rx = 0;
706         ring_idx = 0;
707         nb_rxd = 0;
708         idx = 0;
709
710         rxq = rx_queue;
711         hw = rxq->hw;
712
713         rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
714
715         if (unlikely(rxq->stopped)) {
716                 PMD_RX_LOG(DEBUG, "Rx queue is stopped.");
717                 return 0;
718         }
719
720         while (rcd->gen == rxq->comp_ring.gen) {
721                 struct rte_mbuf *newm;
722
723                 if (nb_rx >= nb_pkts)
724                         break;
725
726                 newm = rte_mbuf_raw_alloc(rxq->mp);
727                 if (unlikely(newm == NULL)) {
728                         PMD_RX_LOG(ERR, "Error allocating mbuf");
729                         rxq->stats.rx_buf_alloc_failure++;
730                         break;
731                 }
732
733                 idx = rcd->rxdIdx;
734                 ring_idx = vmxnet3_get_ring_idx(hw, rcd->rqID);
735                 rxd = (Vmxnet3_RxDesc *)rxq->cmd_ring[ring_idx].base + idx;
736                 RTE_SET_USED(rxd); /* used only for assert when enabled */
737                 rbi = rxq->cmd_ring[ring_idx].buf_info + idx;
738
739                 PMD_RX_LOG(DEBUG, "rxd idx: %d ring idx: %d.", idx, ring_idx);
740
741                 RTE_ASSERT(rcd->len <= rxd->len);
742                 RTE_ASSERT(rbi->m);
743
744                 /* Get the packet buffer pointer from buf_info */
745                 rxm = rbi->m;
746
747                 /* Clear descriptor associated buf_info to be reused */
748                 rbi->m = NULL;
749                 rbi->bufPA = 0;
750
751                 /* Update the index that we received a packet */
752                 rxq->cmd_ring[ring_idx].next2comp = idx;
753
754                 /* For RCD with EOP set, check if there is frame error */
755                 if (unlikely(rcd->eop && rcd->err)) {
756                         rxq->stats.drop_total++;
757                         rxq->stats.drop_err++;
758
759                         if (!rcd->fcs) {
760                                 rxq->stats.drop_fcs++;
761                                 PMD_RX_LOG(ERR, "Recv packet dropped due to frame err.");
762                         }
763                         PMD_RX_LOG(ERR, "Error in received packet rcd#:%d rxd:%d",
764                                    (int)(rcd - (struct Vmxnet3_RxCompDesc *)
765                                          rxq->comp_ring.base), rcd->rxdIdx);
766                         rte_pktmbuf_free_seg(rxm);
767                         if (rxq->start_seg) {
768                                 struct rte_mbuf *start = rxq->start_seg;
769
770                                 rxq->start_seg = NULL;
771                                 rte_pktmbuf_free(start);
772                         }
773                         goto rcd_done;
774                 }
775
776                 /* Initialize newly received packet buffer */
777                 rxm->port = rxq->port_id;
778                 rxm->nb_segs = 1;
779                 rxm->next = NULL;
780                 rxm->pkt_len = (uint16_t)rcd->len;
781                 rxm->data_len = (uint16_t)rcd->len;
782                 rxm->data_off = RTE_PKTMBUF_HEADROOM;
783                 rxm->ol_flags = 0;
784                 rxm->vlan_tci = 0;
785
786                 /*
787                  * If this is the first buffer of the received packet,
788                  * set the pointer to the first mbuf of the packet
789                  * Otherwise, update the total length and the number of segments
790                  * of the current scattered packet, and update the pointer to
791                  * the last mbuf of the current packet.
792                  */
793                 if (rcd->sop) {
794                         RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_HEAD);
795
796                         if (unlikely(rcd->len == 0)) {
797                                 RTE_ASSERT(rcd->eop);
798
799                                 PMD_RX_LOG(DEBUG,
800                                            "Rx buf was skipped. rxring[%d][%d])",
801                                            ring_idx, idx);
802                                 rte_pktmbuf_free_seg(rxm);
803                                 goto rcd_done;
804                         }
805
806                         if (vmxnet3_rx_data_ring(hw, rcd->rqID)) {
807                                 uint8_t *rdd = rxq->data_ring.base +
808                                         idx * rxq->data_desc_size;
809
810                                 RTE_ASSERT(VMXNET3_VERSION_GE_3(hw));
811                                 rte_memcpy(rte_pktmbuf_mtod(rxm, char *),
812                                            rdd, rcd->len);
813                         }
814
815                         rxq->start_seg = rxm;
816                         vmxnet3_rx_offload(hw, rcd, rxm, 1);
817                 } else {
818                         struct rte_mbuf *start = rxq->start_seg;
819
820                         RTE_ASSERT(rxd->btype == VMXNET3_RXD_BTYPE_BODY);
821
822                         start->pkt_len += rxm->data_len;
823                         start->nb_segs++;
824
825                         rxq->last_seg->next = rxm;
826                 }
827                 rxq->last_seg = rxm;
828
829                 if (rcd->eop) {
830                         struct rte_mbuf *start = rxq->start_seg;
831
832                         vmxnet3_rx_offload(hw, rcd, start, 0);
833                         rx_pkts[nb_rx++] = start;
834                         rxq->start_seg = NULL;
835                 }
836
837 rcd_done:
838                 rxq->cmd_ring[ring_idx].next2comp = idx;
839                 VMXNET3_INC_RING_IDX_ONLY(rxq->cmd_ring[ring_idx].next2comp,
840                                           rxq->cmd_ring[ring_idx].size);
841
842                 /* It's time to renew descriptors */
843                 vmxnet3_renew_desc(rxq, ring_idx, newm);
844                 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
845                         VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
846                                                rxq->cmd_ring[ring_idx].next2fill);
847                 }
848
849                 /* Advance to the next descriptor in comp_ring */
850                 vmxnet3_comp_ring_adv_next2proc(&rxq->comp_ring);
851
852                 rcd = &rxq->comp_ring.base[rxq->comp_ring.next2proc].rcd;
853                 nb_rxd++;
854                 if (nb_rxd > rxq->cmd_ring[0].size) {
855                         PMD_RX_LOG(ERR, "Used up quota of receiving packets,"
856                                    " relinquish control.");
857                         break;
858                 }
859         }
860
861         if (unlikely(nb_rxd == 0)) {
862                 uint32_t avail;
863                 for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
864                         avail = vmxnet3_cmd_ring_desc_avail(&rxq->cmd_ring[ring_idx]);
865                         if (unlikely(avail > 0)) {
866                                 /* try to alloc new buf and renew descriptors */
867                                 vmxnet3_post_rx_bufs(rxq, ring_idx);
868                         }
869                 }
870                 if (unlikely(rxq->shared->ctrl.updateRxProd)) {
871                         for (ring_idx = 0; ring_idx < VMXNET3_RX_CMDRING_SIZE; ring_idx++) {
872                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[ring_idx] + (rxq->queue_id * VMXNET3_REG_ALIGN),
873                                                        rxq->cmd_ring[ring_idx].next2fill);
874                         }
875                 }
876         }
877
878         return nb_rx;
879 }
880
881 int
882 vmxnet3_dev_tx_queue_setup(struct rte_eth_dev *dev,
883                            uint16_t queue_idx,
884                            uint16_t nb_desc,
885                            unsigned int socket_id,
886                            const struct rte_eth_txconf *tx_conf)
887 {
888         struct vmxnet3_hw *hw = dev->data->dev_private;
889         const struct rte_memzone *mz;
890         struct vmxnet3_tx_queue *txq;
891         struct vmxnet3_cmd_ring *ring;
892         struct vmxnet3_comp_ring *comp_ring;
893         struct vmxnet3_data_ring *data_ring;
894         int size;
895
896         PMD_INIT_FUNC_TRACE();
897
898         if ((tx_conf->txq_flags & ETH_TXQ_FLAGS_NOXSUMSCTP) !=
899             ETH_TXQ_FLAGS_NOXSUMSCTP) {
900                 PMD_INIT_LOG(ERR, "SCTP checksum offload not supported");
901                 return -EINVAL;
902         }
903
904         txq = rte_zmalloc("ethdev_tx_queue", sizeof(struct vmxnet3_tx_queue),
905                           RTE_CACHE_LINE_SIZE);
906         if (txq == NULL) {
907                 PMD_INIT_LOG(ERR, "Can not allocate tx queue structure");
908                 return -ENOMEM;
909         }
910
911         txq->queue_id = queue_idx;
912         txq->port_id = dev->data->port_id;
913         txq->shared = NULL; /* set in vmxnet3_setup_driver_shared() */
914         txq->hw = hw;
915         txq->qid = queue_idx;
916         txq->stopped = TRUE;
917         txq->txdata_desc_size = hw->txdata_desc_size;
918
919         ring = &txq->cmd_ring;
920         comp_ring = &txq->comp_ring;
921         data_ring = &txq->data_ring;
922
923         /* Tx vmxnet ring length should be between 512-4096 */
924         if (nb_desc < VMXNET3_DEF_TX_RING_SIZE) {
925                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Min: %u",
926                              VMXNET3_DEF_TX_RING_SIZE);
927                 return -EINVAL;
928         } else if (nb_desc > VMXNET3_TX_RING_MAX_SIZE) {
929                 PMD_INIT_LOG(ERR, "VMXNET3 Tx Ring Size Max: %u",
930                              VMXNET3_TX_RING_MAX_SIZE);
931                 return -EINVAL;
932         } else {
933                 ring->size = nb_desc;
934                 ring->size &= ~VMXNET3_RING_SIZE_MASK;
935         }
936         comp_ring->size = data_ring->size = ring->size;
937
938         /* Tx vmxnet rings structure initialization*/
939         ring->next2fill = 0;
940         ring->next2comp = 0;
941         ring->gen = VMXNET3_INIT_GEN;
942         comp_ring->next2proc = 0;
943         comp_ring->gen = VMXNET3_INIT_GEN;
944
945         size = sizeof(struct Vmxnet3_TxDesc) * ring->size;
946         size += sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size;
947         size += txq->txdata_desc_size * data_ring->size;
948
949         mz = rte_eth_dma_zone_reserve(dev, "txdesc", queue_idx, size,
950                                       VMXNET3_RING_BA_ALIGN, socket_id);
951         if (mz == NULL) {
952                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
953                 return -ENOMEM;
954         }
955         txq->mz = mz;
956         memset(mz->addr, 0, mz->len);
957
958         /* cmd_ring initialization */
959         ring->base = mz->addr;
960         ring->basePA = mz->iova;
961
962         /* comp_ring initialization */
963         comp_ring->base = ring->base + ring->size;
964         comp_ring->basePA = ring->basePA +
965                 (sizeof(struct Vmxnet3_TxDesc) * ring->size);
966
967         /* data_ring initialization */
968         data_ring->base = (Vmxnet3_TxDataDesc *)(comp_ring->base + comp_ring->size);
969         data_ring->basePA = comp_ring->basePA +
970                         (sizeof(struct Vmxnet3_TxCompDesc) * comp_ring->size);
971
972         /* cmd_ring0 buf_info allocation */
973         ring->buf_info = rte_zmalloc("tx_ring_buf_info",
974                                      ring->size * sizeof(vmxnet3_buf_info_t), RTE_CACHE_LINE_SIZE);
975         if (ring->buf_info == NULL) {
976                 PMD_INIT_LOG(ERR, "ERROR: Creating tx_buf_info structure");
977                 return -ENOMEM;
978         }
979
980         /* Update the data portion with txq */
981         dev->data->tx_queues[queue_idx] = txq;
982
983         return 0;
984 }
985
986 int
987 vmxnet3_dev_rx_queue_setup(struct rte_eth_dev *dev,
988                            uint16_t queue_idx,
989                            uint16_t nb_desc,
990                            unsigned int socket_id,
991                            __rte_unused const struct rte_eth_rxconf *rx_conf,
992                            struct rte_mempool *mp)
993 {
994         const struct rte_memzone *mz;
995         struct vmxnet3_rx_queue *rxq;
996         struct vmxnet3_hw *hw = dev->data->dev_private;
997         struct vmxnet3_cmd_ring *ring0, *ring1, *ring;
998         struct vmxnet3_comp_ring *comp_ring;
999         struct vmxnet3_rx_data_ring *data_ring;
1000         int size;
1001         uint8_t i;
1002         char mem_name[32];
1003
1004         PMD_INIT_FUNC_TRACE();
1005
1006         rxq = rte_zmalloc("ethdev_rx_queue", sizeof(struct vmxnet3_rx_queue),
1007                           RTE_CACHE_LINE_SIZE);
1008         if (rxq == NULL) {
1009                 PMD_INIT_LOG(ERR, "Can not allocate rx queue structure");
1010                 return -ENOMEM;
1011         }
1012
1013         rxq->mp = mp;
1014         rxq->queue_id = queue_idx;
1015         rxq->port_id = dev->data->port_id;
1016         rxq->shared = NULL; /* set in vmxnet3_setup_driver_shared() */
1017         rxq->hw = hw;
1018         rxq->qid1 = queue_idx;
1019         rxq->qid2 = queue_idx + hw->num_rx_queues;
1020         rxq->data_ring_qid = queue_idx + 2 * hw->num_rx_queues;
1021         rxq->data_desc_size = hw->rxdata_desc_size;
1022         rxq->stopped = TRUE;
1023
1024         ring0 = &rxq->cmd_ring[0];
1025         ring1 = &rxq->cmd_ring[1];
1026         comp_ring = &rxq->comp_ring;
1027         data_ring = &rxq->data_ring;
1028
1029         /* Rx vmxnet rings length should be between 256-4096 */
1030         if (nb_desc < VMXNET3_DEF_RX_RING_SIZE) {
1031                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Min: 256");
1032                 return -EINVAL;
1033         } else if (nb_desc > VMXNET3_RX_RING_MAX_SIZE) {
1034                 PMD_INIT_LOG(ERR, "VMXNET3 Rx Ring Size Max: 4096");
1035                 return -EINVAL;
1036         } else {
1037                 ring0->size = nb_desc;
1038                 ring0->size &= ~VMXNET3_RING_SIZE_MASK;
1039                 ring1->size = ring0->size;
1040         }
1041
1042         comp_ring->size = ring0->size + ring1->size;
1043         data_ring->size = ring0->size;
1044
1045         /* Rx vmxnet rings structure initialization */
1046         ring0->next2fill = 0;
1047         ring1->next2fill = 0;
1048         ring0->next2comp = 0;
1049         ring1->next2comp = 0;
1050         ring0->gen = VMXNET3_INIT_GEN;
1051         ring1->gen = VMXNET3_INIT_GEN;
1052         comp_ring->next2proc = 0;
1053         comp_ring->gen = VMXNET3_INIT_GEN;
1054
1055         size = sizeof(struct Vmxnet3_RxDesc) * (ring0->size + ring1->size);
1056         size += sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1057         if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size)
1058                 size += rxq->data_desc_size * data_ring->size;
1059
1060         mz = rte_eth_dma_zone_reserve(dev, "rxdesc", queue_idx, size,
1061                                       VMXNET3_RING_BA_ALIGN, socket_id);
1062         if (mz == NULL) {
1063                 PMD_INIT_LOG(ERR, "ERROR: Creating queue descriptors zone");
1064                 return -ENOMEM;
1065         }
1066         rxq->mz = mz;
1067         memset(mz->addr, 0, mz->len);
1068
1069         /* cmd_ring0 initialization */
1070         ring0->base = mz->addr;
1071         ring0->basePA = mz->iova;
1072
1073         /* cmd_ring1 initialization */
1074         ring1->base = ring0->base + ring0->size;
1075         ring1->basePA = ring0->basePA + sizeof(struct Vmxnet3_RxDesc) * ring0->size;
1076
1077         /* comp_ring initialization */
1078         comp_ring->base = ring1->base + ring1->size;
1079         comp_ring->basePA = ring1->basePA + sizeof(struct Vmxnet3_RxDesc) *
1080                 ring1->size;
1081
1082         /* data_ring initialization */
1083         if (VMXNET3_VERSION_GE_3(hw) && rxq->data_desc_size) {
1084                 data_ring->base =
1085                         (uint8_t *)(comp_ring->base + comp_ring->size);
1086                 data_ring->basePA = comp_ring->basePA +
1087                         sizeof(struct Vmxnet3_RxCompDesc) * comp_ring->size;
1088         }
1089
1090         /* cmd_ring0-cmd_ring1 buf_info allocation */
1091         for (i = 0; i < VMXNET3_RX_CMDRING_SIZE; i++) {
1092
1093                 ring = &rxq->cmd_ring[i];
1094                 ring->rid = i;
1095                 snprintf(mem_name, sizeof(mem_name), "rx_ring_%d_buf_info", i);
1096
1097                 ring->buf_info = rte_zmalloc(mem_name,
1098                                              ring->size * sizeof(vmxnet3_buf_info_t),
1099                                              RTE_CACHE_LINE_SIZE);
1100                 if (ring->buf_info == NULL) {
1101                         PMD_INIT_LOG(ERR, "ERROR: Creating rx_buf_info structure");
1102                         return -ENOMEM;
1103                 }
1104         }
1105
1106         /* Update the data portion with rxq */
1107         dev->data->rx_queues[queue_idx] = rxq;
1108
1109         return 0;
1110 }
1111
1112 /*
1113  * Initializes Receive Unit
1114  * Load mbufs in rx queue in advance
1115  */
1116 int
1117 vmxnet3_dev_rxtx_init(struct rte_eth_dev *dev)
1118 {
1119         struct vmxnet3_hw *hw = dev->data->dev_private;
1120
1121         int i, ret;
1122         uint8_t j;
1123
1124         PMD_INIT_FUNC_TRACE();
1125
1126         for (i = 0; i < hw->num_rx_queues; i++) {
1127                 vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i];
1128
1129                 for (j = 0; j < VMXNET3_RX_CMDRING_SIZE; j++) {
1130                         /* Passing 0 as alloc_num will allocate full ring */
1131                         ret = vmxnet3_post_rx_bufs(rxq, j);
1132                         if (ret <= 0) {
1133                                 PMD_INIT_LOG(ERR,
1134                                              "ERROR: Posting Rxq: %d buffers ring: %d",
1135                                              i, j);
1136                                 return -ret;
1137                         }
1138                         /*
1139                          * Updating device with the index:next2fill to fill the
1140                          * mbufs for coming packets.
1141                          */
1142                         if (unlikely(rxq->shared->ctrl.updateRxProd)) {
1143                                 VMXNET3_WRITE_BAR0_REG(hw, rxprod_reg[j] + (rxq->queue_id * VMXNET3_REG_ALIGN),
1144                                                        rxq->cmd_ring[j].next2fill);
1145                         }
1146                 }
1147                 rxq->stopped = FALSE;
1148                 rxq->start_seg = NULL;
1149         }
1150
1151         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1152                 struct vmxnet3_tx_queue *txq = dev->data->tx_queues[i];
1153
1154                 txq->stopped = FALSE;
1155         }
1156
1157         return 0;
1158 }
1159
1160 static uint8_t rss_intel_key[40] = {
1161         0x6D, 0x5A, 0x56, 0xDA, 0x25, 0x5B, 0x0E, 0xC2,
1162         0x41, 0x67, 0x25, 0x3D, 0x43, 0xA3, 0x8F, 0xB0,
1163         0xD0, 0xCA, 0x2B, 0xCB, 0xAE, 0x7B, 0x30, 0xB4,
1164         0x77, 0xCB, 0x2D, 0xA3, 0x80, 0x30, 0xF2, 0x0C,
1165         0x6A, 0x42, 0xB7, 0x3B, 0xBE, 0xAC, 0x01, 0xFA,
1166 };
1167
1168 /*
1169  * Configure RSS feature
1170  */
1171 int
1172 vmxnet3_rss_configure(struct rte_eth_dev *dev)
1173 {
1174         struct vmxnet3_hw *hw = dev->data->dev_private;
1175         struct VMXNET3_RSSConf *dev_rss_conf;
1176         struct rte_eth_rss_conf *port_rss_conf;
1177         uint64_t rss_hf;
1178         uint8_t i, j;
1179
1180         PMD_INIT_FUNC_TRACE();
1181
1182         dev_rss_conf = hw->rss_conf;
1183         port_rss_conf = &dev->data->dev_conf.rx_adv_conf.rss_conf;
1184
1185         /* loading hashFunc */
1186         dev_rss_conf->hashFunc = VMXNET3_RSS_HASH_FUNC_TOEPLITZ;
1187         /* loading hashKeySize */
1188         dev_rss_conf->hashKeySize = VMXNET3_RSS_MAX_KEY_SIZE;
1189         /* loading indTableSize: Must not exceed VMXNET3_RSS_MAX_IND_TABLE_SIZE (128)*/
1190         dev_rss_conf->indTableSize = (uint16_t)(hw->num_rx_queues * 4);
1191
1192         if (port_rss_conf->rss_key == NULL) {
1193                 /* Default hash key */
1194                 port_rss_conf->rss_key = rss_intel_key;
1195         }
1196
1197         /* loading hashKey */
1198         memcpy(&dev_rss_conf->hashKey[0], port_rss_conf->rss_key,
1199                dev_rss_conf->hashKeySize);
1200
1201         /* loading indTable */
1202         for (i = 0, j = 0; i < dev_rss_conf->indTableSize; i++, j++) {
1203                 if (j == dev->data->nb_rx_queues)
1204                         j = 0;
1205                 dev_rss_conf->indTable[i] = j;
1206         }
1207
1208         /* loading hashType */
1209         dev_rss_conf->hashType = 0;
1210         rss_hf = port_rss_conf->rss_hf & VMXNET3_RSS_OFFLOAD_ALL;
1211         if (rss_hf & ETH_RSS_IPV4)
1212                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV4;
1213         if (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP)
1214                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV4;
1215         if (rss_hf & ETH_RSS_IPV6)
1216                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_IPV6;
1217         if (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP)
1218                 dev_rss_conf->hashType |= VMXNET3_RSS_HASH_TYPE_TCP_IPV6;
1219
1220         return VMXNET3_SUCCESS;
1221 }