4 * Copyright 2015 6WIND S.A.
5 * Copyright 2015 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
41 /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
43 #pragma GCC diagnostic ignored "-pedantic"
45 #include <infiniband/verbs.h>
47 #pragma GCC diagnostic error "-pedantic"
50 /* DPDK headers don't like -pedantic. */
52 #pragma GCC diagnostic ignored "-pedantic"
55 #include <rte_malloc.h>
56 #include <rte_ethdev.h>
57 #include <rte_common.h>
59 #pragma GCC diagnostic error "-pedantic"
63 #include "mlx5_rxtx.h"
64 #include "mlx5_utils.h"
65 #include "mlx5_defs.h"
68 * Allocate RX queue elements with scattered packets support.
71 * Pointer to RX queue structure.
73 * Number of elements to allocate.
75 * If not NULL, fetch buffers from this array instead of allocating them
76 * with rte_pktmbuf_alloc().
79 * 0 on success, errno value on failure.
82 rxq_alloc_elts_sp(struct rxq *rxq, unsigned int elts_n,
83 struct rte_mbuf **pool)
86 struct rxq_elt_sp (*elts)[elts_n] =
87 rte_calloc_socket("RXQ elements", 1, sizeof(*elts), 0,
92 ERROR("%p: can't allocate packets array", (void *)rxq);
96 /* For each WR (packet). */
97 for (i = 0; (i != elts_n); ++i) {
99 struct rxq_elt_sp *elt = &(*elts)[i];
100 struct ibv_recv_wr *wr = &elt->wr;
101 struct ibv_sge (*sges)[RTE_DIM(elt->sges)] = &elt->sges;
103 /* These two arrays must have the same size. */
104 assert(RTE_DIM(elt->sges) == RTE_DIM(elt->bufs));
107 wr->next = &(*elts)[(i + 1)].wr;
108 wr->sg_list = &(*sges)[0];
109 wr->num_sge = RTE_DIM(*sges);
110 /* For each SGE (segment). */
111 for (j = 0; (j != RTE_DIM(elt->bufs)); ++j) {
112 struct ibv_sge *sge = &(*sges)[j];
113 struct rte_mbuf *buf;
118 rte_pktmbuf_reset(buf);
120 buf = rte_pktmbuf_alloc(rxq->mp);
122 assert(pool == NULL);
123 ERROR("%p: empty mbuf pool", (void *)rxq);
128 /* Headroom is reserved by rte_pktmbuf_alloc(). */
129 assert(DATA_OFF(buf) == RTE_PKTMBUF_HEADROOM);
130 /* Buffer is supposed to be empty. */
131 assert(rte_pktmbuf_data_len(buf) == 0);
132 assert(rte_pktmbuf_pkt_len(buf) == 0);
133 /* sge->addr must be able to store a pointer. */
134 assert(sizeof(sge->addr) >= sizeof(uintptr_t));
136 /* The first SGE keeps its headroom. */
137 sge->addr = rte_pktmbuf_mtod(buf, uintptr_t);
138 sge->length = (buf->buf_len -
139 RTE_PKTMBUF_HEADROOM);
141 /* Subsequent SGEs lose theirs. */
142 assert(DATA_OFF(buf) == RTE_PKTMBUF_HEADROOM);
143 SET_DATA_OFF(buf, 0);
144 sge->addr = (uintptr_t)buf->buf_addr;
145 sge->length = buf->buf_len;
147 sge->lkey = rxq->mr->lkey;
148 /* Redundant check for tailroom. */
149 assert(sge->length == rte_pktmbuf_tailroom(buf));
152 /* The last WR pointer must be NULL. */
153 (*elts)[(i - 1)].wr.next = NULL;
154 DEBUG("%p: allocated and configured %u WRs (%zu segments)",
155 (void *)rxq, elts_n, (elts_n * RTE_DIM((*elts)[0].sges)));
156 rxq->elts_n = elts_n;
163 assert(pool == NULL);
164 for (i = 0; (i != RTE_DIM(*elts)); ++i) {
166 struct rxq_elt_sp *elt = &(*elts)[i];
168 for (j = 0; (j != RTE_DIM(elt->bufs)); ++j) {
169 struct rte_mbuf *buf = elt->bufs[j];
172 rte_pktmbuf_free_seg(buf);
177 DEBUG("%p: failed, freed everything", (void *)rxq);
183 * Free RX queue elements with scattered packets support.
186 * Pointer to RX queue structure.
189 rxq_free_elts_sp(struct rxq *rxq)
192 unsigned int elts_n = rxq->elts_n;
193 struct rxq_elt_sp (*elts)[elts_n] = rxq->elts.sp;
195 DEBUG("%p: freeing WRs", (void *)rxq);
200 for (i = 0; (i != RTE_DIM(*elts)); ++i) {
202 struct rxq_elt_sp *elt = &(*elts)[i];
204 for (j = 0; (j != RTE_DIM(elt->bufs)); ++j) {
205 struct rte_mbuf *buf = elt->bufs[j];
208 rte_pktmbuf_free_seg(buf);
215 * Allocate RX queue elements.
218 * Pointer to RX queue structure.
220 * Number of elements to allocate.
222 * If not NULL, fetch buffers from this array instead of allocating them
223 * with rte_pktmbuf_alloc().
226 * 0 on success, errno value on failure.
229 rxq_alloc_elts(struct rxq *rxq, unsigned int elts_n, struct rte_mbuf **pool)
232 struct rxq_elt (*elts)[elts_n] =
233 rte_calloc_socket("RXQ elements", 1, sizeof(*elts), 0,
238 ERROR("%p: can't allocate packets array", (void *)rxq);
242 /* For each WR (packet). */
243 for (i = 0; (i != elts_n); ++i) {
244 struct rxq_elt *elt = &(*elts)[i];
245 struct ibv_recv_wr *wr = &elt->wr;
246 struct ibv_sge *sge = &(*elts)[i].sge;
247 struct rte_mbuf *buf;
252 rte_pktmbuf_reset(buf);
254 buf = rte_pktmbuf_alloc(rxq->mp);
256 assert(pool == NULL);
257 ERROR("%p: empty mbuf pool", (void *)rxq);
261 /* Configure WR. Work request ID contains its own index in
262 * the elts array and the offset between SGE buffer header and
264 WR_ID(wr->wr_id).id = i;
265 WR_ID(wr->wr_id).offset =
266 (((uintptr_t)buf->buf_addr + RTE_PKTMBUF_HEADROOM) -
268 wr->next = &(*elts)[(i + 1)].wr;
271 /* Headroom is reserved by rte_pktmbuf_alloc(). */
272 assert(DATA_OFF(buf) == RTE_PKTMBUF_HEADROOM);
273 /* Buffer is supposed to be empty. */
274 assert(rte_pktmbuf_data_len(buf) == 0);
275 assert(rte_pktmbuf_pkt_len(buf) == 0);
276 /* sge->addr must be able to store a pointer. */
277 assert(sizeof(sge->addr) >= sizeof(uintptr_t));
278 /* SGE keeps its headroom. */
279 sge->addr = (uintptr_t)
280 ((uint8_t *)buf->buf_addr + RTE_PKTMBUF_HEADROOM);
281 sge->length = (buf->buf_len - RTE_PKTMBUF_HEADROOM);
282 sge->lkey = rxq->mr->lkey;
283 /* Redundant check for tailroom. */
284 assert(sge->length == rte_pktmbuf_tailroom(buf));
285 /* Make sure elts index and SGE mbuf pointer can be deduced
287 if ((WR_ID(wr->wr_id).id != i) ||
288 ((void *)((uintptr_t)sge->addr -
289 WR_ID(wr->wr_id).offset) != buf)) {
290 ERROR("%p: cannot store index and offset in WR ID",
293 rte_pktmbuf_free(buf);
298 /* The last WR pointer must be NULL. */
299 (*elts)[(i - 1)].wr.next = NULL;
300 DEBUG("%p: allocated and configured %u single-segment WRs",
301 (void *)rxq, elts_n);
302 rxq->elts_n = elts_n;
304 rxq->elts.no_sp = elts;
309 assert(pool == NULL);
310 for (i = 0; (i != RTE_DIM(*elts)); ++i) {
311 struct rxq_elt *elt = &(*elts)[i];
312 struct rte_mbuf *buf;
314 if (elt->sge.addr == 0)
316 assert(WR_ID(elt->wr.wr_id).id == i);
317 buf = (void *)((uintptr_t)elt->sge.addr -
318 WR_ID(elt->wr.wr_id).offset);
319 rte_pktmbuf_free_seg(buf);
323 DEBUG("%p: failed, freed everything", (void *)rxq);
329 * Free RX queue elements.
332 * Pointer to RX queue structure.
335 rxq_free_elts(struct rxq *rxq)
338 unsigned int elts_n = rxq->elts_n;
339 struct rxq_elt (*elts)[elts_n] = rxq->elts.no_sp;
341 DEBUG("%p: freeing WRs", (void *)rxq);
343 rxq->elts.no_sp = NULL;
346 for (i = 0; (i != RTE_DIM(*elts)); ++i) {
347 struct rxq_elt *elt = &(*elts)[i];
348 struct rte_mbuf *buf;
350 if (elt->sge.addr == 0)
352 assert(WR_ID(elt->wr.wr_id).id == i);
353 buf = (void *)((uintptr_t)elt->sge.addr -
354 WR_ID(elt->wr.wr_id).offset);
355 rte_pktmbuf_free_seg(buf);
361 * Clean up a RX queue.
363 * Destroy objects, free allocated memory and reset the structure for reuse.
366 * Pointer to RX queue structure.
369 rxq_cleanup(struct rxq *rxq)
371 struct ibv_exp_release_intf_params params;
373 DEBUG("cleaning up %p", (void *)rxq);
375 rxq_free_elts_sp(rxq);
378 if (rxq->if_qp != NULL) {
379 assert(rxq->priv != NULL);
380 assert(rxq->priv->ctx != NULL);
381 assert(rxq->qp != NULL);
382 params = (struct ibv_exp_release_intf_params){
385 claim_zero(ibv_exp_release_intf(rxq->priv->ctx,
389 if (rxq->if_cq != NULL) {
390 assert(rxq->priv != NULL);
391 assert(rxq->priv->ctx != NULL);
392 assert(rxq->cq != NULL);
393 params = (struct ibv_exp_release_intf_params){
396 claim_zero(ibv_exp_release_intf(rxq->priv->ctx,
400 if (rxq->qp != NULL) {
401 rxq_promiscuous_disable(rxq);
402 rxq_allmulticast_disable(rxq);
403 rxq_mac_addrs_del(rxq);
404 claim_zero(ibv_destroy_qp(rxq->qp));
407 claim_zero(ibv_destroy_cq(rxq->cq));
408 if (rxq->rd != NULL) {
409 struct ibv_exp_destroy_res_domain_attr attr = {
413 assert(rxq->priv != NULL);
414 assert(rxq->priv->ctx != NULL);
415 claim_zero(ibv_exp_destroy_res_domain(rxq->priv->ctx,
420 claim_zero(ibv_dereg_mr(rxq->mr));
421 memset(rxq, 0, sizeof(*rxq));
425 * Allocate a Queue Pair.
426 * Optionally setup inline receive if supported.
429 * Pointer to private structure.
431 * Completion queue to associate with QP.
433 * Number of descriptors in QP (hint only).
436 * QP pointer or NULL in case of error.
438 static struct ibv_qp *
439 rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc,
440 struct ibv_exp_res_domain *rd)
442 struct ibv_exp_qp_init_attr attr = {
443 /* CQ to be associated with the send queue. */
445 /* CQ to be associated with the receive queue. */
448 /* Max number of outstanding WRs. */
449 .max_recv_wr = ((priv->device_attr.max_qp_wr < desc) ?
450 priv->device_attr.max_qp_wr :
452 /* Max number of scatter/gather elements in a WR. */
453 .max_recv_sge = ((priv->device_attr.max_sge <
455 priv->device_attr.max_sge :
458 .qp_type = IBV_QPT_RAW_PACKET,
459 .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
460 IBV_EXP_QP_INIT_ATTR_RES_DOMAIN),
465 return ibv_exp_create_qp(priv->ctx, &attr);
471 * Allocate a RSS Queue Pair.
472 * Optionally setup inline receive if supported.
475 * Pointer to private structure.
477 * Completion queue to associate with QP.
479 * Number of descriptors in QP (hint only).
481 * If nonzero, create a parent QP, otherwise a child.
484 * QP pointer or NULL in case of error.
486 static struct ibv_qp *
487 rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc,
488 int parent, struct ibv_exp_res_domain *rd)
490 struct ibv_exp_qp_init_attr attr = {
491 /* CQ to be associated with the send queue. */
493 /* CQ to be associated with the receive queue. */
496 /* Max number of outstanding WRs. */
497 .max_recv_wr = ((priv->device_attr.max_qp_wr < desc) ?
498 priv->device_attr.max_qp_wr :
500 /* Max number of scatter/gather elements in a WR. */
501 .max_recv_sge = ((priv->device_attr.max_sge <
503 priv->device_attr.max_sge :
506 .qp_type = IBV_QPT_RAW_PACKET,
507 .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
508 IBV_EXP_QP_INIT_ATTR_RES_DOMAIN |
509 IBV_EXP_QP_INIT_ATTR_QPG),
515 attr.qpg.qpg_type = IBV_EXP_QPG_PARENT;
516 /* TSS isn't necessary. */
517 attr.qpg.parent_attrib.tss_child_count = 0;
518 attr.qpg.parent_attrib.rss_child_count = priv->rxqs_n;
519 DEBUG("initializing parent RSS queue");
521 attr.qpg.qpg_type = IBV_EXP_QPG_CHILD_RX;
522 attr.qpg.qpg_parent = priv->rxq_parent.qp;
523 DEBUG("initializing child RSS queue");
525 return ibv_exp_create_qp(priv->ctx, &attr);
528 #endif /* RSS_SUPPORT */
531 * Reconfigure a RX queue with new parameters.
533 * rxq_rehash() does not allocate mbufs, which, if not done from the right
534 * thread (such as a control thread), may corrupt the pool.
535 * In case of failure, the queue is left untouched.
538 * Pointer to Ethernet device structure.
543 * 0 on success, errno value on failure.
546 rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq)
548 struct priv *priv = rxq->priv;
549 struct rxq tmpl = *rxq;
552 struct rte_mbuf **pool;
554 struct ibv_exp_qp_attr mod;
555 struct ibv_recv_wr *bad_wr;
557 int parent = (rxq == &priv->rxq_parent);
560 ERROR("%p: cannot rehash parent queue %p",
561 (void *)dev, (void *)rxq);
564 DEBUG("%p: rehashing queue %p", (void *)dev, (void *)rxq);
565 /* Number of descriptors and mbufs currently allocated. */
566 desc_n = (tmpl.elts_n * (tmpl.sp ? MLX5_PMD_SGE_WR_N : 1));
568 /* Enable scattered packets support for this queue if necessary. */
569 if ((dev->data->dev_conf.rxmode.jumbo_frame) &&
570 (dev->data->dev_conf.rxmode.max_rx_pkt_len >
571 (tmpl.mb_len - RTE_PKTMBUF_HEADROOM))) {
573 desc_n /= MLX5_PMD_SGE_WR_N;
576 DEBUG("%p: %s scattered packets support (%u WRs)",
577 (void *)dev, (tmpl.sp ? "enabling" : "disabling"), desc_n);
578 /* If scatter mode is the same as before, nothing to do. */
579 if (tmpl.sp == rxq->sp) {
580 DEBUG("%p: nothing to do", (void *)dev);
583 /* Remove attached flows if RSS is disabled (no parent queue). */
585 rxq_allmulticast_disable(&tmpl);
586 rxq_promiscuous_disable(&tmpl);
587 rxq_mac_addrs_del(&tmpl);
588 /* Update original queue in case of failure. */
589 rxq->allmulti_flow = tmpl.allmulti_flow;
590 rxq->promisc_flow = tmpl.promisc_flow;
591 memcpy(rxq->mac_flow, tmpl.mac_flow, sizeof(rxq->mac_flow));
593 /* From now on, any failure will render the queue unusable.
594 * Reinitialize QP. */
595 mod = (struct ibv_exp_qp_attr){ .qp_state = IBV_QPS_RESET };
596 err = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE);
598 ERROR("%p: cannot reset QP: %s", (void *)dev, strerror(err));
602 err = ibv_resize_cq(tmpl.cq, desc_n);
604 ERROR("%p: cannot resize CQ: %s", (void *)dev, strerror(err));
608 mod = (struct ibv_exp_qp_attr){
609 /* Move the QP to this state. */
610 .qp_state = IBV_QPS_INIT,
611 /* Primary port number. */
612 .port_num = priv->port
614 err = ibv_exp_modify_qp(tmpl.qp, &mod,
617 (parent ? IBV_EXP_QP_GROUP_RSS : 0) |
618 #endif /* RSS_SUPPORT */
621 ERROR("%p: QP state to IBV_QPS_INIT failed: %s",
622 (void *)dev, strerror(err));
626 /* Reconfigure flows. Do not care for errors. */
629 rxq_mac_addrs_add(&tmpl);
630 if (priv->started && priv->promisc_req)
631 rxq_promiscuous_enable(&tmpl);
632 if (priv->started && priv->allmulti_req)
633 rxq_allmulticast_enable(&tmpl);
634 /* Update original queue in case of failure. */
635 rxq->allmulti_flow = tmpl.allmulti_flow;
636 rxq->promisc_flow = tmpl.promisc_flow;
637 memcpy(rxq->mac_flow, tmpl.mac_flow, sizeof(rxq->mac_flow));
640 pool = rte_malloc(__func__, (mbuf_n * sizeof(*pool)), 0);
642 ERROR("%p: cannot allocate memory", (void *)dev);
645 /* Snatch mbufs from original queue. */
648 struct rxq_elt_sp (*elts)[rxq->elts_n] = rxq->elts.sp;
650 for (i = 0; (i != RTE_DIM(*elts)); ++i) {
651 struct rxq_elt_sp *elt = &(*elts)[i];
654 for (j = 0; (j != RTE_DIM(elt->bufs)); ++j) {
655 assert(elt->bufs[j] != NULL);
656 pool[k++] = elt->bufs[j];
660 struct rxq_elt (*elts)[rxq->elts_n] = rxq->elts.no_sp;
662 for (i = 0; (i != RTE_DIM(*elts)); ++i) {
663 struct rxq_elt *elt = &(*elts)[i];
664 struct rte_mbuf *buf = (void *)
665 ((uintptr_t)elt->sge.addr -
666 WR_ID(elt->wr.wr_id).offset);
668 assert(WR_ID(elt->wr.wr_id).id == i);
675 assert((void *)&tmpl.elts.sp == (void *)&tmpl.elts.no_sp);
677 rxq_alloc_elts_sp(&tmpl, desc_n, pool) :
678 rxq_alloc_elts(&tmpl, desc_n, pool));
680 ERROR("%p: cannot reallocate WRs, aborting", (void *)dev);
685 assert(tmpl.elts_n == desc_n);
686 assert(tmpl.elts.sp != NULL);
688 /* Clean up original data. */
690 rte_free(rxq->elts.sp);
693 err = ibv_post_recv(tmpl.qp,
695 &(*tmpl.elts.sp)[0].wr :
696 &(*tmpl.elts.no_sp)[0].wr),
699 ERROR("%p: ibv_post_recv() failed for WR %p: %s",
705 mod = (struct ibv_exp_qp_attr){
706 .qp_state = IBV_QPS_RTR
708 err = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE);
710 ERROR("%p: QP state to IBV_QPS_RTR failed: %s",
711 (void *)dev, strerror(err));
719 * Configure a RX queue.
722 * Pointer to Ethernet device structure.
724 * Pointer to RX queue structure.
726 * Number of descriptors to configure in queue.
728 * NUMA socket on which memory must be allocated.
730 * Thresholds parameters.
732 * Memory pool for buffer allocations.
735 * 0 on success, errno value on failure.
738 rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc,
739 unsigned int socket, const struct rte_eth_rxconf *conf,
740 struct rte_mempool *mp)
742 struct priv *priv = dev->data->dev_private;
748 struct ibv_exp_qp_attr mod;
750 struct ibv_exp_query_intf_params params;
751 struct ibv_exp_cq_init_attr cq;
752 struct ibv_exp_res_domain_init_attr rd;
754 enum ibv_exp_query_intf_status status;
755 struct ibv_recv_wr *bad_wr;
756 struct rte_mbuf *buf;
758 int parent = (rxq == &priv->rxq_parent);
760 (void)conf; /* Thresholds configuration (ignored). */
762 * If this is a parent queue, hardware must support RSS and
763 * RSS must be enabled.
765 assert((!parent) || ((priv->hw_rss) && (priv->rss)));
767 /* Even if unused, ibv_create_cq() requires at least one
772 if ((desc == 0) || (desc % MLX5_PMD_SGE_WR_N)) {
773 ERROR("%p: invalid number of RX descriptors (must be a"
774 " multiple of %d)", (void *)dev, MLX5_PMD_SGE_WR_N);
777 /* Get mbuf length. */
778 buf = rte_pktmbuf_alloc(mp);
780 ERROR("%p: unable to allocate mbuf", (void *)dev);
783 tmpl.mb_len = buf->buf_len;
784 assert((rte_pktmbuf_headroom(buf) +
785 rte_pktmbuf_tailroom(buf)) == tmpl.mb_len);
786 assert(rte_pktmbuf_headroom(buf) == RTE_PKTMBUF_HEADROOM);
787 rte_pktmbuf_free(buf);
788 /* Enable scattered packets support for this queue if necessary. */
789 if ((dev->data->dev_conf.rxmode.jumbo_frame) &&
790 (dev->data->dev_conf.rxmode.max_rx_pkt_len >
791 (tmpl.mb_len - RTE_PKTMBUF_HEADROOM))) {
793 desc /= MLX5_PMD_SGE_WR_N;
795 DEBUG("%p: %s scattered packets support (%u WRs)",
796 (void *)dev, (tmpl.sp ? "enabling" : "disabling"), desc);
797 /* Use the entire RX mempool as the memory region. */
798 tmpl.mr = ibv_reg_mr(priv->pd,
799 (void *)mp->elt_va_start,
800 (mp->elt_va_end - mp->elt_va_start),
801 (IBV_ACCESS_LOCAL_WRITE |
802 IBV_ACCESS_REMOTE_WRITE));
803 if (tmpl.mr == NULL) {
805 ERROR("%p: MR creation failure: %s",
806 (void *)dev, strerror(ret));
810 attr.rd = (struct ibv_exp_res_domain_init_attr){
811 .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL |
812 IBV_EXP_RES_DOMAIN_MSG_MODEL),
813 .thread_model = IBV_EXP_THREAD_SINGLE,
814 .msg_model = IBV_EXP_MSG_HIGH_BW,
816 tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd);
817 if (tmpl.rd == NULL) {
819 ERROR("%p: RD creation failure: %s",
820 (void *)dev, strerror(ret));
823 attr.cq = (struct ibv_exp_cq_init_attr){
824 .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN,
825 .res_domain = tmpl.rd,
827 tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq);
828 if (tmpl.cq == NULL) {
830 ERROR("%p: CQ creation failure: %s",
831 (void *)dev, strerror(ret));
834 DEBUG("priv->device_attr.max_qp_wr is %d",
835 priv->device_attr.max_qp_wr);
836 DEBUG("priv->device_attr.max_sge is %d",
837 priv->device_attr.max_sge);
840 tmpl.qp = rxq_setup_qp_rss(priv, tmpl.cq, desc, parent,
843 #endif /* RSS_SUPPORT */
844 tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc, tmpl.rd);
845 if (tmpl.qp == NULL) {
846 ret = (errno ? errno : EINVAL);
847 ERROR("%p: QP creation failure: %s",
848 (void *)dev, strerror(ret));
851 mod = (struct ibv_exp_qp_attr){
852 /* Move the QP to this state. */
853 .qp_state = IBV_QPS_INIT,
854 /* Primary port number. */
855 .port_num = priv->port
857 ret = ibv_exp_modify_qp(tmpl.qp, &mod,
860 (parent ? IBV_EXP_QP_GROUP_RSS : 0) |
861 #endif /* RSS_SUPPORT */
864 ERROR("%p: QP state to IBV_QPS_INIT failed: %s",
865 (void *)dev, strerror(ret));
868 if ((parent) || (!priv->rss)) {
869 /* Configure MAC and broadcast addresses. */
870 ret = rxq_mac_addrs_add(&tmpl);
872 ERROR("%p: QP flow attachment failed: %s",
873 (void *)dev, strerror(ret));
877 /* Allocate descriptors for RX queues, except for the RSS parent. */
881 ret = rxq_alloc_elts_sp(&tmpl, desc, NULL);
883 ret = rxq_alloc_elts(&tmpl, desc, NULL);
885 ERROR("%p: RXQ allocation failed: %s",
886 (void *)dev, strerror(ret));
889 ret = ibv_post_recv(tmpl.qp,
891 &(*tmpl.elts.sp)[0].wr :
892 &(*tmpl.elts.no_sp)[0].wr),
895 ERROR("%p: ibv_post_recv() failed for WR %p: %s",
902 mod = (struct ibv_exp_qp_attr){
903 .qp_state = IBV_QPS_RTR
905 ret = ibv_exp_modify_qp(tmpl.qp, &mod, IBV_EXP_QP_STATE);
907 ERROR("%p: QP state to IBV_QPS_RTR failed: %s",
908 (void *)dev, strerror(ret));
912 tmpl.port_id = dev->data->port_id;
913 DEBUG("%p: RTE port ID: %u", (void *)rxq, tmpl.port_id);
914 attr.params = (struct ibv_exp_query_intf_params){
915 .intf_scope = IBV_EXP_INTF_GLOBAL,
916 .intf = IBV_EXP_INTF_CQ,
919 tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status);
920 if (tmpl.if_cq == NULL) {
921 ERROR("%p: CQ interface family query failed with status %d",
922 (void *)dev, status);
925 attr.params = (struct ibv_exp_query_intf_params){
926 .intf_scope = IBV_EXP_INTF_GLOBAL,
927 .intf = IBV_EXP_INTF_QP_BURST,
930 tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status);
931 if (tmpl.if_qp == NULL) {
932 ERROR("%p: QP interface family query failed with status %d",
933 (void *)dev, status);
936 /* Clean up rxq in case we're reinitializing it. */
937 DEBUG("%p: cleaning-up old rxq just in case", (void *)rxq);
940 DEBUG("%p: rxq updated with %p", (void *)rxq, (void *)&tmpl);
950 * DPDK callback to configure a RX queue.
953 * Pointer to Ethernet device structure.
957 * Number of descriptors to configure in queue.
959 * NUMA socket on which memory must be allocated.
961 * Thresholds parameters.
963 * Memory pool for buffer allocations.
966 * 0 on success, negative errno value on failure.
969 mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
970 unsigned int socket, const struct rte_eth_rxconf *conf,
971 struct rte_mempool *mp)
973 struct priv *priv = dev->data->dev_private;
974 struct rxq *rxq = (*priv->rxqs)[idx];
978 DEBUG("%p: configuring queue %u for %u descriptors",
979 (void *)dev, idx, desc);
980 if (idx >= priv->rxqs_n) {
981 ERROR("%p: queue index out of range (%u >= %u)",
982 (void *)dev, idx, priv->rxqs_n);
987 DEBUG("%p: reusing already allocated queue index %u (%p)",
988 (void *)dev, idx, (void *)rxq);
993 (*priv->rxqs)[idx] = NULL;
996 rxq = rte_calloc_socket("RXQ", 1, sizeof(*rxq), 0, socket);
998 ERROR("%p: unable to allocate queue index %u",
1004 ret = rxq_setup(dev, rxq, desc, socket, conf, mp);
1008 rxq->stats.idx = idx;
1009 DEBUG("%p: adding RX queue %p to list",
1010 (void *)dev, (void *)rxq);
1011 (*priv->rxqs)[idx] = rxq;
1012 /* Update receive callback. */
1014 dev->rx_pkt_burst = mlx5_rx_burst_sp;
1016 dev->rx_pkt_burst = mlx5_rx_burst;
1023 * DPDK callback to release a RX queue.
1026 * Generic RX queue pointer.
1029 mlx5_rx_queue_release(void *dpdk_rxq)
1031 struct rxq *rxq = (struct rxq *)dpdk_rxq;
1039 assert(rxq != &priv->rxq_parent);
1040 for (i = 0; (i != priv->rxqs_n); ++i)
1041 if ((*priv->rxqs)[i] == rxq) {
1042 DEBUG("%p: removing RX queue %p from list",
1043 (void *)priv->dev, (void *)rxq);
1044 (*priv->rxqs)[i] = NULL;