X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx4%2Fmlx4.c;h=f4491e7176012ac47c16b252fcd6a4048b884c8c;hb=6ffefc4c8fe0f8dc3ee88aa43e062c4167a27b47;hp=08b1b81c64d8484e55e8cfcc218c89ee2ab11ba9;hpb=c3f23415827dfe7605cc367367e4f33108bf7781;p=dpdk.git diff --git a/drivers/net/mlx4/mlx4.c b/drivers/net/mlx4/mlx4.c index 08b1b81c64..f4491e7176 100644 --- a/drivers/net/mlx4/mlx4.c +++ b/drivers/net/mlx4/mlx4.c @@ -33,8 +33,6 @@ /* * Known limitations: - * - Multiple RX VLAN filters can be configured, but only the first one - * works properly. * - RSS hash key and options cannot be modified. * - Hardware counters aren't implemented. */ @@ -141,14 +139,11 @@ static inline void wr_id_t_check(void) (void)wr_id_t_check; } -/* If raw send operations are available, use them since they are faster. */ -#ifdef SEND_RAW_WR_SUPPORT -typedef struct ibv_send_wr_raw mlx4_send_wr_t; -#define mlx4_post_send ibv_post_send_raw -#else -typedef struct ibv_send_wr mlx4_send_wr_t; -#define mlx4_post_send ibv_post_send -#endif +/* Transpose flags. Useful to convert IBV to DPDK flags. */ +#define TRANSPOSE(val, from, to) \ + (((from) >= (to)) ? \ + (((val) & (from)) / ((from) / (to))) : \ + (((val) & (from)) * ((to) / (from)))) struct mlx4_rxq_stats { unsigned int idx; /**< Mapping index. */ @@ -190,14 +185,15 @@ struct rxq { struct ibv_mr *mr; /* Memory Region (for mp). */ struct ibv_cq *cq; /* Completion Queue. */ struct ibv_qp *qp; /* Queue Pair. */ + struct ibv_exp_qp_burst_family *if_qp; /* QP burst interface. */ + struct ibv_exp_cq_family *if_cq; /* CQ interface. */ /* - * There is exactly one flow configured per MAC address. Each flow - * may contain several specifications, one per configured VLAN ID. + * Each VLAN ID requires a separate flow steering rule. */ BITFIELD_DECLARE(mac_configured, uint32_t, MLX4_MAX_MAC_ADDRESSES); - struct ibv_exp_flow *mac_flow[MLX4_MAX_MAC_ADDRESSES]; - struct ibv_exp_flow *promisc_flow; /* Promiscuous flow. */ - struct ibv_exp_flow *allmulti_flow; /* Multicast flow. */ + struct ibv_flow *mac_flow[MLX4_MAX_MAC_ADDRESSES][MLX4_MAX_VLAN_IDS]; + struct ibv_flow *promisc_flow; /* Promiscuous flow. */ + struct ibv_flow *allmulti_flow; /* Multicast flow. */ unsigned int port_id; /* Port ID for incoming packets. */ unsigned int elts_n; /* (*elts)[] length. */ unsigned int elts_head; /* Current index in (*elts)[]. */ @@ -206,16 +202,17 @@ struct rxq { struct rxq_elt (*no_sp)[]; /* RX elements. */ } elts; unsigned int sp:1; /* Use scattered RX elements. */ + unsigned int csum:1; /* Enable checksum offloading. */ + unsigned int csum_l2tun:1; /* Same for L2 tunnels. */ uint32_t mb_len; /* Length of a mp-issued mbuf. */ struct mlx4_rxq_stats stats; /* RX queue counters. */ unsigned int socket; /* CPU socket ID for allocations. */ + struct ibv_exp_res_domain *rd; /* Resource Domain. */ }; /* TX element. */ struct txq_elt { - mlx4_send_wr_t wr; /* Work Request. */ - struct ibv_sge sges[MLX4_PMD_SGE_WR_N]; /* Scatter/Gather Elements. */ - /* mbuf pointer is derived from WR_ID(wr.wr_id).offset. */ + struct rte_mbuf *buf; }; /* Linear buffer type. It is used when transmitting buffers with too many @@ -236,6 +233,8 @@ struct txq { } mp2mr[MLX4_PMD_TX_MP_CACHE]; /* MP to MR translation table. */ struct ibv_cq *cq; /* Completion Queue. */ struct ibv_qp *qp; /* Queue Pair. */ + struct ibv_exp_qp_burst_family *if_qp; /* QP burst interface. */ + struct ibv_exp_cq_family *if_cq; /* CQ interface. */ #if MLX4_PMD_MAX_INLINE > 0 uint32_t max_inline; /* Max inline send size <= MLX4_PMD_MAX_INLINE. */ #endif @@ -244,17 +243,19 @@ struct txq { unsigned int elts_head; /* Current index in (*elts)[]. */ unsigned int elts_tail; /* First element awaiting completion. */ unsigned int elts_comp; /* Number of completion requests. */ + unsigned int elts_comp_cd; /* Countdown for next completion request. */ + unsigned int elts_comp_cd_init; /* Initial value for countdown. */ struct mlx4_txq_stats stats; /* TX queue counters. */ linear_t (*elts_linear)[]; /* Linearized buffers. */ struct ibv_mr *mr_linear; /* Memory Region for linearized buffers. */ unsigned int socket; /* CPU socket ID for allocations. */ + struct ibv_exp_res_domain *rd; /* Resource Domain. */ }; struct priv { struct rte_eth_dev *dev; /* Ethernet device. */ struct ibv_context *ctx; /* Verbs context. */ struct ibv_device_attr device_attr; /* Device properties. */ - struct ibv_port_attr port_attr; /* Physical port properties. */ struct ibv_pd *pd; /* Protection Domain. */ /* * MAC addresses array and configuration bit-field. @@ -273,11 +274,12 @@ struct priv { uint8_t port; /* Physical port number. */ unsigned int started:1; /* Device started, flows enabled. */ unsigned int promisc:1; /* Device in promiscuous mode. */ - unsigned int promisc_ok:1; /* Promiscuous flow is supported. */ unsigned int allmulti:1; /* Device receives all multicast packets. */ unsigned int hw_qpg:1; /* QP groups are supported. */ unsigned int hw_tss:1; /* TSS is supported. */ unsigned int hw_rss:1; /* RSS is supported. */ + unsigned int hw_csum:1; /* Checksum offload is supported. */ + unsigned int hw_csum_l2tun:1; /* Same for L2 tunnels. */ unsigned int rss:1; /* RSS is enabled. */ unsigned int vf:1; /* This is a VF device. */ #ifdef INLINE_RECV @@ -798,14 +800,8 @@ txq_alloc_elts(struct txq *txq, unsigned int elts_n) } for (i = 0; (i != elts_n); ++i) { struct txq_elt *elt = &(*elts)[i]; - mlx4_send_wr_t *wr = &elt->wr; - /* Configure WR. */ - WR_ID(wr->wr_id).id = i; - WR_ID(wr->wr_id).offset = 0; - wr->sg_list = &elt->sges[0]; - wr->opcode = IBV_WR_SEND; - /* Other fields are updated during TX. */ + elt->buf = NULL; } DEBUG("%p: allocated and configured %u WRs", (void *)txq, elts_n); txq->elts_n = elts_n; @@ -813,6 +809,12 @@ txq_alloc_elts(struct txq *txq, unsigned int elts_n) txq->elts_head = 0; txq->elts_tail = 0; txq->elts_comp = 0; + /* Request send completion every MLX4_PMD_TX_PER_COMP_REQ packets or + * at least 4 times per ring. */ + txq->elts_comp_cd_init = + ((MLX4_PMD_TX_PER_COMP_REQ < (elts_n / 4)) ? + MLX4_PMD_TX_PER_COMP_REQ : (elts_n / 4)); + txq->elts_comp_cd = txq->elts_comp_cd_init; txq->elts_linear = elts_linear; txq->mr_linear = mr_linear; assert(ret == 0); @@ -858,10 +860,9 @@ txq_free_elts(struct txq *txq) for (i = 0; (i != elemof(*elts)); ++i) { struct txq_elt *elt = &(*elts)[i]; - if (WR_ID(elt->wr.wr_id).offset == 0) + if (elt->buf == NULL) continue; - rte_pktmbuf_free((void *)((uintptr_t)elt->sges[0].addr - - WR_ID(elt->wr.wr_id).offset)); + rte_pktmbuf_free(elt->buf); } rte_free(elts); } @@ -878,14 +879,48 @@ txq_free_elts(struct txq *txq) static void txq_cleanup(struct txq *txq) { + struct ibv_exp_release_intf_params params; size_t i; DEBUG("cleaning up %p", (void *)txq); txq_free_elts(txq); + if (txq->if_qp != NULL) { + assert(txq->priv != NULL); + assert(txq->priv->ctx != NULL); + assert(txq->qp != NULL); + params = (struct ibv_exp_release_intf_params){ + .comp_mask = 0, + }; + claim_zero(ibv_exp_release_intf(txq->priv->ctx, + txq->if_qp, + ¶ms)); + } + if (txq->if_cq != NULL) { + assert(txq->priv != NULL); + assert(txq->priv->ctx != NULL); + assert(txq->cq != NULL); + params = (struct ibv_exp_release_intf_params){ + .comp_mask = 0, + }; + claim_zero(ibv_exp_release_intf(txq->priv->ctx, + txq->if_cq, + ¶ms)); + } if (txq->qp != NULL) claim_zero(ibv_destroy_qp(txq->qp)); if (txq->cq != NULL) claim_zero(ibv_destroy_cq(txq->cq)); + if (txq->rd != NULL) { + struct ibv_exp_destroy_res_domain_attr attr = { + .comp_mask = 0, + }; + + assert(txq->priv != NULL); + assert(txq->priv->ctx != NULL); + claim_zero(ibv_exp_destroy_res_domain(txq->priv->ctx, + txq->rd, + &attr)); + } for (i = 0; (i != elemof(txq->mp2mr)); ++i) { if (txq->mp2mr[i].mp == NULL) break; @@ -899,9 +934,9 @@ txq_cleanup(struct txq *txq) * Manage TX completions. * * When sending a burst, mlx4_tx_burst() posts several WRs. - * To improve performance, a completion event is only required for the last of - * them. Doing so discards completion information for other WRs, but this - * information would not be used anyway. + * To improve performance, a completion event is only required once every + * MLX4_PMD_TX_PER_COMP_REQ sends. Doing so discards completion information + * for other WRs, but this information would not be used anyway. * * @param txq * Pointer to TX queue structure. @@ -913,9 +948,8 @@ static int txq_complete(struct txq *txq) { unsigned int elts_comp = txq->elts_comp; - unsigned int elts_tail; + unsigned int elts_tail = txq->elts_tail; const unsigned int elts_n = txq->elts_n; - struct ibv_wc wcs[elts_comp]; int wcs_n; if (unlikely(elts_comp == 0)) @@ -924,7 +958,7 @@ txq_complete(struct txq *txq) DEBUG("%p: processing %u work requests completions", (void *)txq, elts_comp); #endif - wcs_n = ibv_poll_cq(txq->cq, elts_comp, wcs); + wcs_n = txq->if_cq->poll_cnt(txq->cq, elts_comp); if (unlikely(wcs_n == 0)) return 0; if (unlikely(wcs_n < 0)) { @@ -935,17 +969,12 @@ txq_complete(struct txq *txq) elts_comp -= wcs_n; assert(elts_comp <= txq->elts_comp); /* - * Work Completion ID contains the associated element index in - * (*txq->elts)[]. Since WCs are returned in order, we only need to - * look at the last WC to clear older Work Requests. - * * Assume WC status is successful as nothing can be done about it * anyway. */ - elts_tail = WR_ID(wcs[wcs_n - 1].wr_id).id; - /* Consume the last WC. */ - if (++elts_tail >= elts_n) - elts_tail = 0; + elts_tail += wcs_n * txq->elts_comp_cd_init; + if (elts_tail >= elts_n) + elts_tail -= elts_n; txq->elts_tail = elts_tail; txq->elts_comp = elts_comp; return 0; @@ -1010,6 +1039,8 @@ txq_mp2mr(struct txq *txq, struct rte_mempool *mp) return txq->mp2mr[i].lkey; } +#if MLX4_PMD_SGE_WR_N > 1 + /** * Copy scattered mbuf contents to a single linear buffer. * @@ -1042,6 +1073,120 @@ linearize_mbuf(linear_t *linear, struct rte_mbuf *buf) return size; } +/** + * Handle scattered buffers for mlx4_tx_burst(). + * + * @param txq + * TX queue structure. + * @param segs + * Number of segments in buf. + * @param elt + * TX queue element to fill. + * @param[in] buf + * Buffer to process. + * @param elts_head + * Index of the linear buffer to use if necessary (normally txq->elts_head). + * @param[out] sges + * Array filled with SGEs on success. + * + * @return + * A structure containing the processed packet size in bytes and the + * number of SGEs. Both fields are set to (unsigned int)-1 in case of + * failure. + */ +static struct tx_burst_sg_ret { + unsigned int length; + unsigned int num; +} +tx_burst_sg(struct txq *txq, unsigned int segs, struct txq_elt *elt, + struct rte_mbuf *buf, unsigned int elts_head, + struct ibv_sge (*sges)[MLX4_PMD_SGE_WR_N]) +{ + unsigned int sent_size = 0; + unsigned int j; + int linearize = 0; + + /* When there are too many segments, extra segments are + * linearized in the last SGE. */ + if (unlikely(segs > elemof(*sges))) { + segs = (elemof(*sges) - 1); + linearize = 1; + } + /* Update element. */ + elt->buf = buf; + /* Register segments as SGEs. */ + for (j = 0; (j != segs); ++j) { + struct ibv_sge *sge = &(*sges)[j]; + uint32_t lkey; + + /* Retrieve Memory Region key for this memory pool. */ + lkey = txq_mp2mr(txq, buf->pool); + if (unlikely(lkey == (uint32_t)-1)) { + /* MR does not exist. */ + DEBUG("%p: unable to get MP <-> MR association", + (void *)txq); + /* Clean up TX element. */ + elt->buf = NULL; + goto stop; + } + /* Update SGE. */ + sge->addr = rte_pktmbuf_mtod(buf, uintptr_t); + if (txq->priv->vf) + rte_prefetch0((volatile void *) + (uintptr_t)sge->addr); + sge->length = DATA_LEN(buf); + sge->lkey = lkey; + sent_size += sge->length; + buf = NEXT(buf); + } + /* If buf is not NULL here and is not going to be linearized, + * nb_segs is not valid. */ + assert(j == segs); + assert((buf == NULL) || (linearize)); + /* Linearize extra segments. */ + if (linearize) { + struct ibv_sge *sge = &(*sges)[segs]; + linear_t *linear = &(*txq->elts_linear)[elts_head]; + unsigned int size = linearize_mbuf(linear, buf); + + assert(segs == (elemof(*sges) - 1)); + if (size == 0) { + /* Invalid packet. */ + DEBUG("%p: packet too large to be linearized.", + (void *)txq); + /* Clean up TX element. */ + elt->buf = NULL; + goto stop; + } + /* If MLX4_PMD_SGE_WR_N is 1, free mbuf immediately. */ + if (elemof(*sges) == 1) { + do { + struct rte_mbuf *next = NEXT(buf); + + rte_pktmbuf_free_seg(buf); + buf = next; + } while (buf != NULL); + elt->buf = NULL; + } + /* Update SGE. */ + sge->addr = (uintptr_t)&(*linear)[0]; + sge->length = size; + sge->lkey = txq->mr_linear->lkey; + sent_size += size; + } + return (struct tx_burst_sg_ret){ + .length = sent_size, + .num = segs, + }; +stop: + return (struct tx_burst_sg_ret){ + .length = -1, + .num = -1, + }; +} + +#endif /* MLX4_PMD_SGE_WR_N > 1 */ + /** * DPDK callback for TX. * @@ -1059,16 +1204,16 @@ static uint16_t mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) { struct txq *txq = (struct txq *)dpdk_txq; - mlx4_send_wr_t head; - mlx4_send_wr_t **wr_next = &head.next; - mlx4_send_wr_t *bad_wr; unsigned int elts_head = txq->elts_head; const unsigned int elts_tail = txq->elts_tail; const unsigned int elts_n = txq->elts_n; + unsigned int elts_comp_cd = txq->elts_comp_cd; + unsigned int elts_comp = 0; unsigned int i; unsigned int max; int err; + assert(elts_comp_cd != 0); txq_complete(txq); max = (elts_n - (elts_head - elts_tail)); if (max > elts_n) @@ -1083,20 +1228,19 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) max = pkts_n; for (i = 0; (i != max); ++i) { struct rte_mbuf *buf = pkts[i]; + unsigned int elts_head_next = + (((elts_head + 1) == elts_n) ? 0 : elts_head + 1); + struct txq_elt *elt_next = &(*txq->elts)[elts_head_next]; struct txq_elt *elt = &(*txq->elts)[elts_head]; - mlx4_send_wr_t *wr = &elt->wr; unsigned int segs = NB_SEGS(buf); -#if (MLX4_PMD_MAX_INLINE > 0) || defined(MLX4_PMD_SOFT_COUNTERS) +#ifdef MLX4_PMD_SOFT_COUNTERS unsigned int sent_size = 0; #endif - unsigned int j; - int linearize = 0; + uint32_t send_flags = 0; /* Clean up old buffer. */ - if (likely(WR_ID(wr->wr_id).offset != 0)) { - struct rte_mbuf *tmp = (void *) - ((uintptr_t)elt->sges[0].addr - - WR_ID(wr->wr_id).offset); + if (likely(elt->buf != NULL)) { + struct rte_mbuf *tmp = elt->buf; /* Faster than rte_pktmbuf_free(). */ do { @@ -1106,43 +1250,37 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) tmp = next; } while (tmp != NULL); } -#ifndef NDEBUG - /* For assert(). */ - WR_ID(wr->wr_id).offset = 0; - for (j = 0; ((int)j < wr->num_sge); ++j) { - elt->sges[j].addr = 0; - elt->sges[j].length = 0; - elt->sges[j].lkey = 0; + /* Request TX completion. */ + if (unlikely(--elts_comp_cd == 0)) { + elts_comp_cd = txq->elts_comp_cd_init; + ++elts_comp; + send_flags |= IBV_EXP_QP_BURST_SIGNALED; } - wr->next = NULL; - wr->num_sge = 0; -#endif - /* Sanity checks, most of which are only relevant with - * debugging enabled. */ - assert(WR_ID(wr->wr_id).id == elts_head); - assert(WR_ID(wr->wr_id).offset == 0); - assert(wr->next == NULL); - assert(wr->sg_list == &elt->sges[0]); - assert(wr->num_sge == 0); - assert(wr->opcode == IBV_WR_SEND); - /* When there are too many segments, extra segments are - * linearized in the last SGE. */ - if (unlikely(segs > elemof(elt->sges))) { - segs = (elemof(elt->sges) - 1); - linearize = 1; + /* Should we enable HW CKSUM offload */ + if (buf->ol_flags & + (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) { + send_flags |= IBV_EXP_QP_BURST_IP_CSUM; + /* HW does not support checksum offloads at arbitrary + * offsets but automatically recognizes the packet + * type. For inner L3/L4 checksums, only VXLAN (UDP) + * tunnels are currently supported. + * + * FIXME: since PKT_TX_UDP_TUNNEL_PKT has been removed, + * the outer packet type is unknown. All we know is + * that the L2 header is of unusual length (not + * ETHER_HDR_LEN with or without 802.1Q header). */ + if ((buf->l2_len != ETHER_HDR_LEN) && + (buf->l2_len != (ETHER_HDR_LEN + 4))) + send_flags |= IBV_EXP_QP_BURST_TUNNEL; } - /* Set WR fields. */ - assert((rte_pktmbuf_mtod(buf, uintptr_t) - - (uintptr_t)buf) <= 0xffff); - WR_ID(wr->wr_id).offset = - (rte_pktmbuf_mtod(buf, uintptr_t) - - (uintptr_t)buf); - wr->num_sge = segs; - /* Register segments as SGEs. */ - for (j = 0; (j != segs); ++j) { - struct ibv_sge *sge = &elt->sges[j]; + if (likely(segs == 1)) { + uintptr_t addr; + uint32_t length; uint32_t lkey; + /* Retrieve buffer information. */ + addr = rte_pktmbuf_mtod(buf, uintptr_t); + length = DATA_LEN(buf); /* Retrieve Memory Region key for this memory pool. */ lkey = txq_mp2mr(txq, buf->pool); if (unlikely(lkey == (uint32_t)-1)) { @@ -1150,104 +1288,64 @@ mlx4_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n) DEBUG("%p: unable to get MP <-> MR" " association", (void *)txq); /* Clean up TX element. */ - WR_ID(elt->wr.wr_id).offset = 0; -#ifndef NDEBUG - /* For assert(). */ - while (j) { - --j; - --sge; - sge->addr = 0; - sge->length = 0; - sge->lkey = 0; - } - wr->num_sge = 0; -#endif + elt->buf = NULL; goto stop; } - /* Sanity checks, only relevant with debugging - * enabled. */ - assert(sge->addr == 0); - assert(sge->length == 0); - assert(sge->lkey == 0); - /* Update SGE. */ - sge->addr = rte_pktmbuf_mtod(buf, uintptr_t); + /* Update element. */ + elt->buf = buf; if (txq->priv->vf) rte_prefetch0((volatile void *) - (uintptr_t)sge->addr); - sge->length = DATA_LEN(buf); - sge->lkey = lkey; -#if (MLX4_PMD_MAX_INLINE > 0) || defined(MLX4_PMD_SOFT_COUNTERS) - sent_size += sge->length; -#endif - buf = NEXT(buf); - } - /* If buf is not NULL here and is not going to be linearized, - * nb_segs is not valid. */ - assert(j == segs); - assert((buf == NULL) || (linearize)); - /* Linearize extra segments. */ - if (linearize) { - struct ibv_sge *sge = &elt->sges[segs]; - linear_t *linear = &(*txq->elts_linear)[elts_head]; - unsigned int size = linearize_mbuf(linear, buf); - - assert(segs == (elemof(elt->sges) - 1)); - if (size == 0) { - /* Invalid packet. */ - DEBUG("%p: packet too large to be linearized.", - (void *)txq); - /* Clean up TX element. */ - WR_ID(elt->wr.wr_id).offset = 0; -#ifndef NDEBUG - /* For assert(). */ - while (j) { - --j; - --sge; - sge->addr = 0; - sge->length = 0; - sge->lkey = 0; - } - wr->num_sge = 0; + (uintptr_t)addr); + RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf); + /* Put packet into send queue. */ +#if MLX4_PMD_MAX_INLINE > 0 + if (length <= txq->max_inline) + err = txq->if_qp->send_pending_inline + (txq->qp, + (void *)addr, + length, + send_flags); + else #endif + err = txq->if_qp->send_pending + (txq->qp, + addr, + length, + lkey, + send_flags); + if (unlikely(err)) goto stop; - } - /* If MLX4_PMD_SGE_WR_N is 1, free mbuf immediately - * and clear offset from WR ID. */ - if (elemof(elt->sges) == 1) { - do { - struct rte_mbuf *next = NEXT(buf); +#ifdef MLX4_PMD_SOFT_COUNTERS + sent_size += length; +#endif + } else { +#if MLX4_PMD_SGE_WR_N > 1 + struct ibv_sge sges[MLX4_PMD_SGE_WR_N]; + struct tx_burst_sg_ret ret; - rte_pktmbuf_free_seg(buf); - buf = next; - } while (buf != NULL); - WR_ID(wr->wr_id).offset = 0; - } - /* Set WR fields and fill SGE with linear buffer. */ - ++wr->num_sge; - /* Sanity checks, only relevant with debugging - * enabled. */ - assert(sge->addr == 0); - assert(sge->length == 0); - assert(sge->lkey == 0); - /* Update SGE. */ - sge->addr = (uintptr_t)&(*linear)[0]; - sge->length = size; - sge->lkey = txq->mr_linear->lkey; -#if (MLX4_PMD_MAX_INLINE > 0) || defined(MLX4_PMD_SOFT_COUNTERS) - sent_size += size; + ret = tx_burst_sg(txq, segs, elt, buf, elts_head, + &sges); + if (ret.length == (unsigned int)-1) + goto stop; + RTE_MBUF_PREFETCH_TO_FREE(elt_next->buf); + /* Put SG list into send queue. */ + err = txq->if_qp->send_pending_sg_list + (txq->qp, + sges, + ret.num, + send_flags); + if (unlikely(err)) + goto stop; +#ifdef MLX4_PMD_SOFT_COUNTERS + sent_size += ret.length; #endif +#else /* MLX4_PMD_SGE_WR_N > 1 */ + DEBUG("%p: TX scattered buffers support not" + " compiled in", (void *)txq); + goto stop; +#endif /* MLX4_PMD_SGE_WR_N > 1 */ } - /* Link WRs together for ibv_post_send(). */ - *wr_next = wr; - wr_next = &wr->next; -#if MLX4_PMD_MAX_INLINE > 0 - if (sent_size <= txq->max_inline) - wr->send_flags = IBV_SEND_INLINE; - else -#endif - wr->send_flags = 0; - if (++elts_head >= elts_n) - elts_head = 0; + elts_head = elts_head_next; #ifdef MLX4_PMD_SOFT_COUNTERS /* Increment sent bytes counter. */ txq->stats.obytes += sent_size; @@ -1261,61 +1359,17 @@ stop: /* Increment sent packets counter. */ txq->stats.opackets += i; #endif - *wr_next = NULL; - /* The last WR is the only one asking for a completion event. */ - containerof(wr_next, mlx4_send_wr_t, next)-> - send_flags |= IBV_SEND_SIGNALED; - err = mlx4_post_send(txq->qp, head.next, &bad_wr); + /* Ring QP doorbell. */ + err = txq->if_qp->send_flush(txq->qp); if (unlikely(err)) { - unsigned int unsent = 0; - - /* An error occurred, completion event is lost. Fix counters. */ - while (bad_wr != NULL) { - struct txq_elt *elt = - containerof(bad_wr, struct txq_elt, wr); - mlx4_send_wr_t *wr = &elt->wr; - mlx4_send_wr_t *next = wr->next; -#if defined(MLX4_PMD_SOFT_COUNTERS) || !defined(NDEBUG) - unsigned int j; -#endif - - assert(wr == bad_wr); - /* Clean up TX element without freeing it, caller - * should take care of this. */ - WR_ID(elt->wr.wr_id).offset = 0; -#ifdef MLX4_PMD_SOFT_COUNTERS - for (j = 0; ((int)j < wr->num_sge); ++j) - txq->stats.obytes -= wr->sg_list[j].length; -#endif - ++unsent; -#ifndef NDEBUG - /* For assert(). */ - for (j = 0; ((int)j < wr->num_sge); ++j) { - elt->sges[j].addr = 0; - elt->sges[j].length = 0; - elt->sges[j].lkey = 0; - } - wr->next = NULL; - wr->num_sge = 0; -#endif - bad_wr = next; - } -#ifdef MLX4_PMD_SOFT_COUNTERS - txq->stats.opackets -= unsent; -#endif - assert(i >= unsent); - i -= unsent; - /* "Unsend" remaining packets. */ - elts_head -= unsent; - if (elts_head >= elts_n) - elts_head += elts_n; - assert(elts_head < elts_n); - DEBUG("%p: mlx4_post_send() failed, %u unprocessed WRs: %s", - (void *)txq, unsent, - ((err <= -1) ? "Internal error" : strerror(err))); - } else - ++txq->elts_comp; + /* A nonzero value is not supposed to be returned. + * Nothing can be done about it. */ + DEBUG("%p: send_flush() failed with error %d", + (void *)txq, err); + } txq->elts_head = elts_head; + txq->elts_comp += elts_comp; + txq->elts_comp_cd = elts_comp_cd; return i; } @@ -1346,20 +1400,41 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, .socket = socket }; union { - struct ibv_qp_init_attr init; + struct ibv_exp_query_intf_params params; + struct ibv_exp_qp_init_attr init; + struct ibv_exp_res_domain_init_attr rd; + struct ibv_exp_cq_init_attr cq; struct ibv_exp_qp_attr mod; } attr; + enum ibv_exp_query_intf_status status; int ret = 0; (void)conf; /* Thresholds configuration (ignored). */ if ((desc == 0) || (desc % MLX4_PMD_SGE_WR_N)) { ERROR("%p: invalid number of TX descriptors (must be a" - " multiple of %d)", (void *)dev, desc); + " multiple of %d)", (void *)dev, MLX4_PMD_SGE_WR_N); return EINVAL; } desc /= MLX4_PMD_SGE_WR_N; /* MRs will be registered in mp2mr[] later. */ - tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0); + attr.rd = (struct ibv_exp_res_domain_init_attr){ + .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | + IBV_EXP_RES_DOMAIN_MSG_MODEL), + .thread_model = IBV_EXP_THREAD_SINGLE, + .msg_model = IBV_EXP_MSG_HIGH_BW, + }; + tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); + if (tmpl.rd == NULL) { + ret = ENOMEM; + ERROR("%p: RD creation failure: %s", + (void *)dev, strerror(ret)); + goto error; + } + attr.cq = (struct ibv_exp_cq_init_attr){ + .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, + .res_domain = tmpl.rd, + }; + tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq); if (tmpl.cq == NULL) { ret = ENOMEM; ERROR("%p: CQ creation failure: %s", @@ -1370,7 +1445,7 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, priv->device_attr.max_qp_wr); DEBUG("priv->device_attr.max_sge is %d", priv->device_attr.max_sge); - attr.init = (struct ibv_qp_init_attr){ + attr.init = (struct ibv_exp_qp_init_attr){ /* CQ to be associated with the send queue. */ .send_cq = tmpl.cq, /* CQ to be associated with the receive queue. */ @@ -1392,9 +1467,13 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, .qp_type = IBV_QPT_RAW_PACKET, /* Do *NOT* enable this, completions events are managed per * TX burst. */ - .sq_sig_all = 0 + .sq_sig_all = 0, + .pd = priv->pd, + .res_domain = tmpl.rd, + .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | + IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), }; - tmpl.qp = ibv_create_qp(priv->pd, &attr.init); + tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init); if (tmpl.qp == NULL) { ret = (errno ? errno : EINVAL); ERROR("%p: QP creation failure: %s", @@ -1440,6 +1519,35 @@ txq_setup(struct rte_eth_dev *dev, struct txq *txq, uint16_t desc, (void *)dev, strerror(ret)); goto error; } + attr.params = (struct ibv_exp_query_intf_params){ + .intf_scope = IBV_EXP_INTF_GLOBAL, + .intf = IBV_EXP_INTF_CQ, + .obj = tmpl.cq, + }; + tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); + if (tmpl.if_cq == NULL) { + ERROR("%p: CQ interface family query failed with status %d", + (void *)dev, status); + goto error; + } + attr.params = (struct ibv_exp_query_intf_params){ + .intf_scope = IBV_EXP_INTF_GLOBAL, + .intf = IBV_EXP_INTF_QP_BURST, + .obj = tmpl.qp, +#ifdef HAVE_EXP_QP_BURST_CREATE_DISABLE_ETH_LOOPBACK + /* MC loopback must be disabled when not using a VF. */ + .family_flags = + (!priv->vf ? + IBV_EXP_QP_BURST_CREATE_DISABLE_ETH_LOOPBACK : + 0), +#endif + }; + tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status); + if (tmpl.if_qp == NULL) { + ERROR("%p: QP interface family query failed with status %d", + (void *)dev, status); + goto error; + } /* Clean up txq in case we're reinitializing it. */ DEBUG("%p: cleaning-up old txq just in case", (void *)txq); txq_cleanup(txq); @@ -1845,15 +1953,17 @@ rxq_free_elts(struct rxq *rxq) } /** - * Unregister a MAC address from a RX queue. + * Delete flow steering rule. * * @param rxq * Pointer to RX queue structure. * @param mac_index * MAC address index. + * @param vlan_index + * VLAN index. */ static void -rxq_mac_addr_del(struct rxq *rxq, unsigned int mac_index) +rxq_del_flow(struct rxq *rxq, unsigned int mac_index, unsigned int vlan_index) { #ifndef NDEBUG struct priv *priv = rxq->priv; @@ -1861,20 +1971,43 @@ rxq_mac_addr_del(struct rxq *rxq, unsigned int mac_index) (const uint8_t (*)[ETHER_ADDR_LEN]) priv->mac[mac_index].addr_bytes; #endif + assert(rxq->mac_flow[mac_index][vlan_index] != NULL); + DEBUG("%p: removing MAC address %02x:%02x:%02x:%02x:%02x:%02x index %u" + " (VLAN ID %" PRIu16 ")", + (void *)rxq, + (*mac)[0], (*mac)[1], (*mac)[2], (*mac)[3], (*mac)[4], (*mac)[5], + mac_index, priv->vlan_filter[vlan_index].id); + claim_zero(ibv_destroy_flow(rxq->mac_flow[mac_index][vlan_index])); + rxq->mac_flow[mac_index][vlan_index] = NULL; +} + +/** + * Unregister a MAC address from a RX queue. + * + * @param rxq + * Pointer to RX queue structure. + * @param mac_index + * MAC address index. + */ +static void +rxq_mac_addr_del(struct rxq *rxq, unsigned int mac_index) +{ + struct priv *priv = rxq->priv; + unsigned int i; + unsigned int vlans = 0; assert(mac_index < elemof(priv->mac)); - if (!BITFIELD_ISSET(rxq->mac_configured, mac_index)) { - assert(rxq->mac_flow[mac_index] == NULL); + if (!BITFIELD_ISSET(rxq->mac_configured, mac_index)) return; + for (i = 0; (i != elemof(priv->vlan_filter)); ++i) { + if (!priv->vlan_filter[i].enabled) + continue; + rxq_del_flow(rxq, mac_index, i); + vlans++; + } + if (!vlans) { + rxq_del_flow(rxq, mac_index, 0); } - DEBUG("%p: removing MAC address %02x:%02x:%02x:%02x:%02x:%02x" - " index %u", - (void *)rxq, - (*mac)[0], (*mac)[1], (*mac)[2], (*mac)[3], (*mac)[4], (*mac)[5], - mac_index); - assert(rxq->mac_flow[mac_index] != NULL); - claim_zero(ibv_exp_destroy_flow(rxq->mac_flow[mac_index])); - rxq->mac_flow[mac_index] = NULL; BITFIELD_RESET(rxq->mac_configured, mac_index); } @@ -1898,111 +2031,76 @@ static int rxq_promiscuous_enable(struct rxq *); static void rxq_promiscuous_disable(struct rxq *); /** - * Register a MAC address in a RX queue. + * Add single flow steering rule. * * @param rxq * Pointer to RX queue structure. * @param mac_index * MAC address index to register. + * @param vlan_index + * VLAN index. Use -1 for a flow without VLAN. * * @return * 0 on success, errno value on failure. */ static int -rxq_mac_addr_add(struct rxq *rxq, unsigned int mac_index) +rxq_add_flow(struct rxq *rxq, unsigned int mac_index, unsigned int vlan_index) { + struct ibv_flow *flow; struct priv *priv = rxq->priv; const uint8_t (*mac)[ETHER_ADDR_LEN] = - (const uint8_t (*)[ETHER_ADDR_LEN]) - priv->mac[mac_index].addr_bytes; - unsigned int vlans = 0; - unsigned int specs = 0; - unsigned int i, j; - struct ibv_exp_flow *flow; - - assert(mac_index < elemof(priv->mac)); - if (BITFIELD_ISSET(rxq->mac_configured, mac_index)) - rxq_mac_addr_del(rxq, mac_index); - /* Number of configured VLANs. */ - for (i = 0; (i != elemof(priv->vlan_filter)); ++i) - if (priv->vlan_filter[i].enabled) - ++vlans; - specs = (vlans ? vlans : 1); + (const uint8_t (*)[ETHER_ADDR_LEN]) + priv->mac[mac_index].addr_bytes; /* Allocate flow specification on the stack. */ - struct ibv_exp_flow_attr data - [1 + - (sizeof(struct ibv_exp_flow_spec_eth[specs]) / - sizeof(struct ibv_exp_flow_attr)) + - !!(sizeof(struct ibv_exp_flow_spec_eth[specs]) % - sizeof(struct ibv_exp_flow_attr))]; - struct ibv_exp_flow_attr *attr = (void *)&data[0]; - struct ibv_exp_flow_spec_eth *spec = (void *)&data[1]; + struct __attribute__((packed)) { + struct ibv_flow_attr attr; + struct ibv_flow_spec_eth spec; + } data; + struct ibv_flow_attr *attr = &data.attr; + struct ibv_flow_spec_eth *spec = &data.spec; + assert(mac_index < elemof(priv->mac)); + assert((vlan_index < elemof(priv->vlan_filter)) || (vlan_index == -1u)); /* * No padding must be inserted by the compiler between attr and spec. * This layout is expected by libibverbs. */ assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec); - *attr = (struct ibv_exp_flow_attr){ - .type = IBV_EXP_FLOW_ATTR_NORMAL, - .num_of_specs = specs, + *attr = (struct ibv_flow_attr){ + .type = IBV_FLOW_ATTR_NORMAL, + .num_of_specs = 1, .port = priv->port, .flags = 0 }; - *spec = (struct ibv_exp_flow_spec_eth){ - .type = IBV_EXP_FLOW_SPEC_ETH, + *spec = (struct ibv_flow_spec_eth){ + .type = IBV_FLOW_SPEC_ETH, .size = sizeof(*spec), .val = { .dst_mac = { (*mac)[0], (*mac)[1], (*mac)[2], (*mac)[3], (*mac)[4], (*mac)[5] - } + }, + .vlan_tag = ((vlan_index != -1u) ? + htons(priv->vlan_filter[vlan_index].id) : + 0), }, .mask = { .dst_mac = "\xff\xff\xff\xff\xff\xff", - .vlan_tag = (vlans ? htons(0xfff) : 0) + .vlan_tag = ((vlan_index != -1u) ? htons(0xfff) : 0), } }; - /* Fill VLAN specifications. */ - for (i = 0, j = 0; (i != elemof(priv->vlan_filter)); ++i) { - if (!priv->vlan_filter[i].enabled) - continue; - assert(j != vlans); - if (j) - spec[j] = spec[0]; - spec[j].val.vlan_tag = htons(priv->vlan_filter[i].id); - ++j; - } DEBUG("%p: adding MAC address %02x:%02x:%02x:%02x:%02x:%02x index %u" - " (%u VLAN(s) configured)", + " (VLAN %s %" PRIu16 ")", (void *)rxq, (*mac)[0], (*mac)[1], (*mac)[2], (*mac)[3], (*mac)[4], (*mac)[5], mac_index, - vlans); + ((vlan_index != -1u) ? "ID" : "index"), + ((vlan_index != -1u) ? priv->vlan_filter[vlan_index].id : -1u)); /* Create related flow. */ errno = 0; - flow = ibv_exp_create_flow(rxq->qp, attr); + flow = ibv_create_flow(rxq->qp, attr); if (flow == NULL) { - int err = errno; - - /* Flow creation failure is not fatal when in DMFS A0 mode. - * Ignore error if promiscuity is already enabled or can be - * enabled. */ - if (priv->promisc_ok) - return 0; - if ((rxq->promisc_flow != NULL) || - (rxq_promiscuous_enable(rxq) == 0)) { - if (rxq->promisc_flow != NULL) - rxq_promiscuous_disable(rxq); - WARN("cannot configure normal flow but promiscuous" - " mode is fine, assuming promiscuous optimization" - " is enabled" - " (options mlx4_core log_num_mgm_entry_size=-7)"); - priv->promisc_ok = 1; - return 0; - } - errno = err; /* It's not clear whether errno is always set in this case. */ ERROR("%p: flow configuration failed, errno=%d: %s", (void *)rxq, errno, @@ -2011,8 +2109,58 @@ rxq_mac_addr_add(struct rxq *rxq, unsigned int mac_index) return errno; return EINVAL; } - assert(rxq->mac_flow[mac_index] == NULL); - rxq->mac_flow[mac_index] = flow; + if (vlan_index == -1u) + vlan_index = 0; + assert(rxq->mac_flow[mac_index][vlan_index] == NULL); + rxq->mac_flow[mac_index][vlan_index] = flow; + return 0; +} + +/** + * Register a MAC address in a RX queue. + * + * @param rxq + * Pointer to RX queue structure. + * @param mac_index + * MAC address index to register. + * + * @return + * 0 on success, errno value on failure. + */ +static int +rxq_mac_addr_add(struct rxq *rxq, unsigned int mac_index) +{ + struct priv *priv = rxq->priv; + unsigned int i; + unsigned int vlans = 0; + int ret; + + assert(mac_index < elemof(priv->mac)); + if (BITFIELD_ISSET(rxq->mac_configured, mac_index)) + rxq_mac_addr_del(rxq, mac_index); + /* Fill VLAN specifications. */ + for (i = 0; (i != elemof(priv->vlan_filter)); ++i) { + if (!priv->vlan_filter[i].enabled) + continue; + /* Create related flow. */ + ret = rxq_add_flow(rxq, mac_index, i); + if (!ret) { + vlans++; + continue; + } + /* Failure, rollback. */ + while (i != 0) + if (priv->vlan_filter[--i].enabled) + rxq_del_flow(rxq, mac_index, i); + assert(ret > 0); + return ret; + } + /* In case there is no VLAN filter. */ + if (!vlans) { + ret = rxq_add_flow(rxq, mac_index, -1); + if (ret) + return ret; + } BITFIELD_SET(rxq->mac_configured, mac_index); return 0; } @@ -2169,9 +2317,9 @@ end: static int rxq_allmulticast_enable(struct rxq *rxq) { - struct ibv_exp_flow *flow; - struct ibv_exp_flow_attr attr = { - .type = IBV_EXP_FLOW_ATTR_MC_DEFAULT, + struct ibv_flow *flow; + struct ibv_flow_attr attr = { + .type = IBV_FLOW_ATTR_MC_DEFAULT, .num_of_specs = 0, .port = rxq->priv->port, .flags = 0 @@ -2181,7 +2329,7 @@ rxq_allmulticast_enable(struct rxq *rxq) if (rxq->allmulti_flow != NULL) return EBUSY; errno = 0; - flow = ibv_exp_create_flow(rxq->qp, &attr); + flow = ibv_create_flow(rxq->qp, &attr); if (flow == NULL) { /* It's not clear whether errno is always set in this case. */ ERROR("%p: flow configuration failed, errno=%d: %s", @@ -2208,7 +2356,7 @@ rxq_allmulticast_disable(struct rxq *rxq) DEBUG("%p: disabling allmulticast mode", (void *)rxq); if (rxq->allmulti_flow == NULL) return; - claim_zero(ibv_exp_destroy_flow(rxq->allmulti_flow)); + claim_zero(ibv_destroy_flow(rxq->allmulti_flow)); rxq->allmulti_flow = NULL; DEBUG("%p: allmulticast mode disabled", (void *)rxq); } @@ -2225,9 +2373,9 @@ rxq_allmulticast_disable(struct rxq *rxq) static int rxq_promiscuous_enable(struct rxq *rxq) { - struct ibv_exp_flow *flow; - struct ibv_exp_flow_attr attr = { - .type = IBV_EXP_FLOW_ATTR_ALL_DEFAULT, + struct ibv_flow *flow; + struct ibv_flow_attr attr = { + .type = IBV_FLOW_ATTR_ALL_DEFAULT, .num_of_specs = 0, .port = rxq->priv->port, .flags = 0 @@ -2239,7 +2387,7 @@ rxq_promiscuous_enable(struct rxq *rxq) if (rxq->promisc_flow != NULL) return EBUSY; errno = 0; - flow = ibv_exp_create_flow(rxq->qp, &attr); + flow = ibv_create_flow(rxq->qp, &attr); if (flow == NULL) { /* It's not clear whether errno is always set in this case. */ ERROR("%p: flow configuration failed, errno=%d: %s", @@ -2268,7 +2416,7 @@ rxq_promiscuous_disable(struct rxq *rxq) DEBUG("%p: disabling promiscuous mode", (void *)rxq); if (rxq->promisc_flow == NULL) return; - claim_zero(ibv_exp_destroy_flow(rxq->promisc_flow)); + claim_zero(ibv_destroy_flow(rxq->promisc_flow)); rxq->promisc_flow = NULL; DEBUG("%p: promiscuous mode disabled", (void *)rxq); } @@ -2284,11 +2432,35 @@ rxq_promiscuous_disable(struct rxq *rxq) static void rxq_cleanup(struct rxq *rxq) { + struct ibv_exp_release_intf_params params; + DEBUG("cleaning up %p", (void *)rxq); if (rxq->sp) rxq_free_elts_sp(rxq); else rxq_free_elts(rxq); + if (rxq->if_qp != NULL) { + assert(rxq->priv != NULL); + assert(rxq->priv->ctx != NULL); + assert(rxq->qp != NULL); + params = (struct ibv_exp_release_intf_params){ + .comp_mask = 0, + }; + claim_zero(ibv_exp_release_intf(rxq->priv->ctx, + rxq->if_qp, + ¶ms)); + } + if (rxq->if_cq != NULL) { + assert(rxq->priv != NULL); + assert(rxq->priv->ctx != NULL); + assert(rxq->cq != NULL); + params = (struct ibv_exp_release_intf_params){ + .comp_mask = 0, + }; + claim_zero(ibv_exp_release_intf(rxq->priv->ctx, + rxq->if_cq, + ¶ms)); + } if (rxq->qp != NULL) { rxq_promiscuous_disable(rxq); rxq_allmulticast_disable(rxq); @@ -2297,11 +2469,71 @@ rxq_cleanup(struct rxq *rxq) } if (rxq->cq != NULL) claim_zero(ibv_destroy_cq(rxq->cq)); + if (rxq->rd != NULL) { + struct ibv_exp_destroy_res_domain_attr attr = { + .comp_mask = 0, + }; + + assert(rxq->priv != NULL); + assert(rxq->priv->ctx != NULL); + claim_zero(ibv_exp_destroy_res_domain(rxq->priv->ctx, + rxq->rd, + &attr)); + } if (rxq->mr != NULL) claim_zero(ibv_dereg_mr(rxq->mr)); memset(rxq, 0, sizeof(*rxq)); } +/** + * Translate RX completion flags to offload flags. + * + * @param[in] rxq + * Pointer to RX queue structure. + * @param flags + * RX completion flags returned by poll_length_flags(). + * + * @return + * Offload flags (ol_flags) for struct rte_mbuf. + */ +static inline uint32_t +rxq_cq_to_ol_flags(const struct rxq *rxq, uint32_t flags) +{ + uint32_t ol_flags; + + ol_flags = + TRANSPOSE(flags, IBV_EXP_CQ_RX_IPV4_PACKET, PKT_RX_IPV4_HDR) | + TRANSPOSE(flags, IBV_EXP_CQ_RX_IPV6_PACKET, PKT_RX_IPV6_HDR); + if (rxq->csum) + ol_flags |= + TRANSPOSE(~flags, + IBV_EXP_CQ_RX_IP_CSUM_OK, + PKT_RX_IP_CKSUM_BAD) | + TRANSPOSE(~flags, + IBV_EXP_CQ_RX_TCP_UDP_CSUM_OK, + PKT_RX_L4_CKSUM_BAD); + /* + * PKT_RX_IP_CKSUM_BAD and PKT_RX_L4_CKSUM_BAD are used in place + * of PKT_RX_EIP_CKSUM_BAD because the latter is not functional + * (its value is 0). + */ + if ((flags & IBV_EXP_CQ_RX_TUNNEL_PACKET) && (rxq->csum_l2tun)) + ol_flags |= + TRANSPOSE(flags, + IBV_EXP_CQ_RX_OUTER_IPV4_PACKET, + PKT_RX_TUNNEL_IPV4_HDR) | + TRANSPOSE(flags, + IBV_EXP_CQ_RX_OUTER_IPV6_PACKET, + PKT_RX_TUNNEL_IPV6_HDR) | + TRANSPOSE(~flags, + IBV_EXP_CQ_RX_OUTER_IP_CSUM_OK, + PKT_RX_IP_CKSUM_BAD) | + TRANSPOSE(~flags, + IBV_EXP_CQ_RX_OUTER_TCP_UDP_CSUM_OK, + PKT_RX_L4_CKSUM_BAD); + return ol_flags; +} + static uint16_t mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n); @@ -2325,64 +2557,80 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) struct rxq_elt_sp (*elts)[rxq->elts_n] = rxq->elts.sp; const unsigned int elts_n = rxq->elts_n; unsigned int elts_head = rxq->elts_head; - struct ibv_wc wcs[pkts_n]; struct ibv_recv_wr head; struct ibv_recv_wr **next = &head.next; struct ibv_recv_wr *bad_wr; - int ret = 0; - int wcs_n; - int i; + unsigned int i; + unsigned int pkts_ret = 0; + int ret; if (unlikely(!rxq->sp)) return mlx4_rx_burst(dpdk_rxq, pkts, pkts_n); if (unlikely(elts == NULL)) /* See RTE_DEV_CMD_SET_MTU. */ return 0; - wcs_n = ibv_poll_cq(rxq->cq, pkts_n, wcs); - if (unlikely(wcs_n == 0)) - return 0; - if (unlikely(wcs_n < 0)) { - DEBUG("rxq=%p, ibv_poll_cq() failed (wc_n=%d)", - (void *)rxq, wcs_n); - return 0; - } - assert(wcs_n <= (int)pkts_n); - /* For each work completion. */ - for (i = 0; (i != wcs_n); ++i) { - struct ibv_wc *wc = &wcs[i]; - uint64_t wr_id = wc->wr_id; - uint32_t len = wc->byte_len; + for (i = 0; (i != pkts_n); ++i) { struct rxq_elt_sp *elt = &(*elts)[elts_head]; struct ibv_recv_wr *wr = &elt->wr; + uint64_t wr_id = wr->wr_id; + unsigned int len; + unsigned int pkt_buf_len; struct rte_mbuf *pkt_buf = NULL; /* Buffer returned in pkts. */ struct rte_mbuf **pkt_buf_next = &pkt_buf; unsigned int seg_headroom = RTE_PKTMBUF_HEADROOM; unsigned int j = 0; + uint32_t flags; /* Sanity checks. */ #ifdef NDEBUG (void)wr_id; #endif assert(wr_id < rxq->elts_n); - assert(wr_id == wr->wr_id); assert(wr->sg_list == elt->sges); assert(wr->num_sge == elemof(elt->sges)); assert(elts_head < rxq->elts_n); assert(rxq->elts_head < rxq->elts_n); - /* Link completed WRs together for repost. */ - *next = wr; - next = &wr->next; - if (unlikely(wc->status != IBV_WC_SUCCESS)) { - /* Whatever, just repost the offending WR. */ - DEBUG("rxq=%p, wr_id=%" PRIu64 ": bad work completion" - " status (%d): %s", - (void *)rxq, wc->wr_id, wc->status, - ibv_wc_status_str(wc->status)); + ret = rxq->if_cq->poll_length_flags(rxq->cq, NULL, NULL, + &flags); + if (unlikely(ret < 0)) { + struct ibv_wc wc; + int wcs_n; + + DEBUG("rxq=%p, poll_length() failed (ret=%d)", + (void *)rxq, ret); + /* ibv_poll_cq() must be used in case of failure. */ + wcs_n = ibv_poll_cq(rxq->cq, 1, &wc); + if (unlikely(wcs_n == 0)) + break; + if (unlikely(wcs_n < 0)) { + DEBUG("rxq=%p, ibv_poll_cq() failed (wcs_n=%d)", + (void *)rxq, wcs_n); + break; + } + assert(wcs_n == 1); + if (unlikely(wc.status != IBV_WC_SUCCESS)) { + /* Whatever, just repost the offending WR. */ + DEBUG("rxq=%p, wr_id=%" PRIu64 ": bad work" + " completion status (%d): %s", + (void *)rxq, wc.wr_id, wc.status, + ibv_wc_status_str(wc.status)); #ifdef MLX4_PMD_SOFT_COUNTERS - /* Increase dropped packets counter. */ - ++rxq->stats.idropped; + /* Increment dropped packets counter. */ + ++rxq->stats.idropped; #endif - goto repost; + /* Link completed WRs together for repost. */ + *next = wr; + next = &wr->next; + goto repost; + } + ret = wc.byte_len; } + if (ret == 0) + break; + len = ret; + pkt_buf_len = len; + /* Link completed WRs together for repost. */ + *next = wr; + next = &wr->next; /* * Replace spent segments with new ones, concatenate and * return them as pkt_buf. @@ -2467,42 +2715,43 @@ mlx4_rx_burst_sp(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) assert(j != 0); NB_SEGS(pkt_buf) = j; PORT(pkt_buf) = rxq->port_id; - PKT_LEN(pkt_buf) = wc->byte_len; - pkt_buf->ol_flags = 0; + PKT_LEN(pkt_buf) = pkt_buf_len; + pkt_buf->ol_flags = rxq_cq_to_ol_flags(rxq, flags); /* Return packet. */ *(pkts++) = pkt_buf; - ++ret; + ++pkts_ret; #ifdef MLX4_PMD_SOFT_COUNTERS /* Increase bytes counter. */ - rxq->stats.ibytes += wc->byte_len; + rxq->stats.ibytes += pkt_buf_len; #endif repost: if (++elts_head >= elts_n) elts_head = 0; continue; } + if (unlikely(i == 0)) + return 0; *next = NULL; /* Repost WRs. */ #ifdef DEBUG_RECV - DEBUG("%p: reposting %d WRs starting from %" PRIu64 " (%p)", - (void *)rxq, wcs_n, wcs[0].wr_id, (void *)head.next); + DEBUG("%p: reposting %d WRs", (void *)rxq, i); #endif - i = ibv_post_recv(rxq->qp, head.next, &bad_wr); - if (unlikely(i)) { + ret = ibv_post_recv(rxq->qp, head.next, &bad_wr); + if (unlikely(ret)) { /* Inability to repost WRs is fatal. */ DEBUG("%p: ibv_post_recv(): failed for WR %p: %s", (void *)rxq->priv, (void *)bad_wr, - strerror(i)); + strerror(ret)); abort(); } rxq->elts_head = elts_head; #ifdef MLX4_PMD_SOFT_COUNTERS /* Increase packets counter. */ - rxq->stats.ipackets += ret; + rxq->stats.ipackets += pkts_ret; #endif - return ret; + return pkts_ret; } /** @@ -2529,58 +2778,66 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) struct rxq_elt (*elts)[rxq->elts_n] = rxq->elts.no_sp; const unsigned int elts_n = rxq->elts_n; unsigned int elts_head = rxq->elts_head; - struct ibv_wc wcs[pkts_n]; - struct ibv_recv_wr head; - struct ibv_recv_wr **next = &head.next; - struct ibv_recv_wr *bad_wr; - int ret = 0; - int wcs_n; - int i; + struct ibv_sge sges[pkts_n]; + unsigned int i; + unsigned int pkts_ret = 0; + int ret; if (unlikely(rxq->sp)) return mlx4_rx_burst_sp(dpdk_rxq, pkts, pkts_n); - wcs_n = ibv_poll_cq(rxq->cq, pkts_n, wcs); - if (unlikely(wcs_n == 0)) - return 0; - if (unlikely(wcs_n < 0)) { - DEBUG("rxq=%p, ibv_poll_cq() failed (wc_n=%d)", - (void *)rxq, wcs_n); - return 0; - } - assert(wcs_n <= (int)pkts_n); - /* For each work completion. */ - for (i = 0; (i != wcs_n); ++i) { - struct ibv_wc *wc = &wcs[i]; - uint64_t wr_id = wc->wr_id; - uint32_t len = wc->byte_len; + for (i = 0; (i != pkts_n); ++i) { struct rxq_elt *elt = &(*elts)[elts_head]; struct ibv_recv_wr *wr = &elt->wr; + uint64_t wr_id = wr->wr_id; + unsigned int len; struct rte_mbuf *seg = (void *)((uintptr_t)elt->sge.addr - WR_ID(wr_id).offset); struct rte_mbuf *rep; + uint32_t flags; /* Sanity checks. */ assert(WR_ID(wr_id).id < rxq->elts_n); - assert(wr_id == wr->wr_id); assert(wr->sg_list == &elt->sge); assert(wr->num_sge == 1); assert(elts_head < rxq->elts_n); assert(rxq->elts_head < rxq->elts_n); - /* Link completed WRs together for repost. */ - *next = wr; - next = &wr->next; - if (unlikely(wc->status != IBV_WC_SUCCESS)) { - /* Whatever, just repost the offending WR. */ - DEBUG("rxq=%p, wr_id=%" PRIu32 ": bad work completion" - " status (%d): %s", - (void *)rxq, WR_ID(wr_id).id, wc->status, - ibv_wc_status_str(wc->status)); + ret = rxq->if_cq->poll_length_flags(rxq->cq, NULL, NULL, + &flags); + if (unlikely(ret < 0)) { + struct ibv_wc wc; + int wcs_n; + + DEBUG("rxq=%p, poll_length() failed (ret=%d)", + (void *)rxq, ret); + /* ibv_poll_cq() must be used in case of failure. */ + wcs_n = ibv_poll_cq(rxq->cq, 1, &wc); + if (unlikely(wcs_n == 0)) + break; + if (unlikely(wcs_n < 0)) { + DEBUG("rxq=%p, ibv_poll_cq() failed (wcs_n=%d)", + (void *)rxq, wcs_n); + break; + } + assert(wcs_n == 1); + if (unlikely(wc.status != IBV_WC_SUCCESS)) { + /* Whatever, just repost the offending WR. */ + DEBUG("rxq=%p, wr_id=%" PRIu64 ": bad work" + " completion status (%d): %s", + (void *)rxq, wc.wr_id, wc.status, + ibv_wc_status_str(wc.status)); #ifdef MLX4_PMD_SOFT_COUNTERS - /* Increase dropped packets counter. */ - ++rxq->stats.idropped; + /* Increment dropped packets counter. */ + ++rxq->stats.idropped; #endif - goto repost; + /* Add SGE to array for repost. */ + sges[i] = elt->sge; + goto repost; + } + ret = wc.byte_len; } + if (ret == 0) + break; + len = ret; /* * Fetch initial bytes of packet descriptor into a * cacheline while allocating rep. @@ -2609,6 +2866,9 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) (uintptr_t)rep); assert(WR_ID(wr->wr_id).id == WR_ID(wr_id).id); + /* Add SGE to array for repost. */ + sges[i] = elt->sge; + /* Update seg information. */ SET_DATA_OFF(seg, RTE_PKTMBUF_HEADROOM); NB_SEGS(seg) = 1; @@ -2616,47 +2876,45 @@ mlx4_rx_burst(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n) NEXT(seg) = NULL; PKT_LEN(seg) = len; DATA_LEN(seg) = len; - seg->ol_flags = 0; + seg->ol_flags = rxq_cq_to_ol_flags(rxq, flags); /* Return packet. */ *(pkts++) = seg; - ++ret; + ++pkts_ret; #ifdef MLX4_PMD_SOFT_COUNTERS /* Increase bytes counter. */ - rxq->stats.ibytes += wc->byte_len; + rxq->stats.ibytes += len; #endif repost: if (++elts_head >= elts_n) elts_head = 0; continue; } - *next = NULL; + if (unlikely(i == 0)) + return 0; /* Repost WRs. */ #ifdef DEBUG_RECV - DEBUG("%p: reposting %d WRs starting from %" PRIu32 " (%p)", - (void *)rxq, wcs_n, WR_ID(wcs[0].wr_id).id, (void *)head.next); + DEBUG("%p: reposting %u WRs", (void *)rxq, i); #endif - i = ibv_post_recv(rxq->qp, head.next, &bad_wr); - if (unlikely(i)) { + ret = rxq->if_qp->recv_burst(rxq->qp, sges, i); + if (unlikely(ret)) { /* Inability to repost WRs is fatal. */ - DEBUG("%p: ibv_post_recv(): failed for WR %p: %s", + DEBUG("%p: recv_burst(): failed (ret=%d)", (void *)rxq->priv, - (void *)bad_wr, - strerror(i)); + ret); abort(); } rxq->elts_head = elts_head; #ifdef MLX4_PMD_SOFT_COUNTERS /* Increase packets counter. */ - rxq->stats.ipackets += ret; + rxq->stats.ipackets += pkts_ret; #endif - return ret; + return pkts_ret; } -#ifdef INLINE_RECV - /** - * Allocate a Queue Pair in case inline receive is supported. + * Allocate a Queue Pair. + * Optionally setup inline receive if supported. * * @param priv * Pointer to private structure. @@ -2669,14 +2927,14 @@ repost: * QP pointer or NULL in case of error. */ static struct ibv_qp * -rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc) +rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc, + struct ibv_exp_res_domain *rd) { struct ibv_exp_qp_init_attr attr = { /* CQ to be associated with the send queue. */ .send_cq = cq, /* CQ to be associated with the receive queue. */ .recv_cq = cq, - .max_inl_recv = priv->inl_recv_size, .cap = { /* Max number of outstanding WRs. */ .max_recv_wr = ((priv->device_attr.max_qp_wr < desc) ? @@ -2689,61 +2947,24 @@ rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc) MLX4_PMD_SGE_WR_N), }, .qp_type = IBV_QPT_RAW_PACKET, - .pd = priv->pd + .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | + IBV_EXP_QP_INIT_ATTR_RES_DOMAIN), + .pd = priv->pd, + .res_domain = rd, }; - attr.comp_mask = IBV_EXP_QP_INIT_ATTR_PD; +#ifdef INLINE_RECV + attr.max_inl_recv = priv->inl_recv_size; attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_INL_RECV; - +#endif return ibv_exp_create_qp(priv->ctx, &attr); } -#else /* INLINE_RECV */ - -/** - * Allocate a Queue Pair. - * - * @param priv - * Pointer to private structure. - * @param cq - * Completion queue to associate with QP. - * @param desc - * Number of descriptors in QP (hint only). - * - * @return - * QP pointer or NULL in case of error. - */ -static struct ibv_qp * -rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc) -{ - struct ibv_qp_init_attr attr = { - /* CQ to be associated with the send queue. */ - .send_cq = cq, - /* CQ to be associated with the receive queue. */ - .recv_cq = cq, - .cap = { - /* Max number of outstanding WRs. */ - .max_recv_wr = ((priv->device_attr.max_qp_wr < desc) ? - priv->device_attr.max_qp_wr : - desc), - /* Max number of scatter/gather elements in a WR. */ - .max_recv_sge = ((priv->device_attr.max_sge < - MLX4_PMD_SGE_WR_N) ? - priv->device_attr.max_sge : - MLX4_PMD_SGE_WR_N), - }, - .qp_type = IBV_QPT_RAW_PACKET - }; - - return ibv_create_qp(priv->pd, &attr); -} - -#endif /* INLINE_RECV */ - #ifdef RSS_SUPPORT /** * Allocate a RSS Queue Pair. + * Optionally setup inline receive if supported. * * @param priv * Pointer to private structure. @@ -2759,16 +2980,13 @@ rxq_setup_qp(struct priv *priv, struct ibv_cq *cq, uint16_t desc) */ static struct ibv_qp * rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc, - int parent) + int parent, struct ibv_exp_res_domain *rd) { struct ibv_exp_qp_init_attr attr = { /* CQ to be associated with the send queue. */ .send_cq = cq, /* CQ to be associated with the receive queue. */ .recv_cq = cq, -#ifdef INLINE_RECV - .max_inl_recv = priv->inl_recv_size, -#endif .cap = { /* Max number of outstanding WRs. */ .max_recv_wr = ((priv->device_attr.max_qp_wr < desc) ? @@ -2782,11 +3000,14 @@ rxq_setup_qp_rss(struct priv *priv, struct ibv_cq *cq, uint16_t desc, }, .qp_type = IBV_QPT_RAW_PACKET, .comp_mask = (IBV_EXP_QP_INIT_ATTR_PD | + IBV_EXP_QP_INIT_ATTR_RES_DOMAIN | IBV_EXP_QP_INIT_ATTR_QPG), - .pd = priv->pd + .pd = priv->pd, + .res_domain = rd, }; #ifdef INLINE_RECV + attr.max_inl_recv = priv->inl_recv_size, attr.comp_mask |= IBV_EXP_QP_INIT_ATTR_INL_RECV; #endif if (parent) { @@ -2843,6 +3064,15 @@ rxq_rehash(struct rte_eth_dev *dev, struct rxq *rxq) /* Number of descriptors and mbufs currently allocated. */ desc_n = (tmpl.elts_n * (tmpl.sp ? MLX4_PMD_SGE_WR_N : 1)); mbuf_n = desc_n; + /* Toggle RX checksum offload if hardware supports it. */ + if (priv->hw_csum) { + tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum; + rxq->csum = tmpl.csum; + } + if (priv->hw_csum_l2tun) { + tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum; + rxq->csum_l2tun = tmpl.csum_l2tun; + } /* Enable scattered packets support for this queue if necessary. */ if ((dev->data->dev_conf.rxmode.jumbo_frame) && (dev->data->dev_conf.rxmode.max_rx_pkt_len > @@ -3027,6 +3257,12 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, .socket = socket }; struct ibv_exp_qp_attr mod; + union { + struct ibv_exp_query_intf_params params; + struct ibv_exp_cq_init_attr cq; + struct ibv_exp_res_domain_init_attr rd; + } attr; + enum ibv_exp_query_intf_status status; struct ibv_recv_wr *bad_wr; struct rte_mbuf *buf; int ret = 0; @@ -3046,7 +3282,7 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, } if ((desc == 0) || (desc % MLX4_PMD_SGE_WR_N)) { ERROR("%p: invalid number of RX descriptors (must be a" - " multiple of %d)", (void *)dev, desc); + " multiple of %d)", (void *)dev, MLX4_PMD_SGE_WR_N); return EINVAL; } /* Get mbuf length. */ @@ -3060,6 +3296,11 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, rte_pktmbuf_tailroom(buf)) == tmpl.mb_len); assert(rte_pktmbuf_headroom(buf) == RTE_PKTMBUF_HEADROOM); rte_pktmbuf_free(buf); + /* Toggle RX checksum offload if hardware supports it. */ + if (priv->hw_csum) + tmpl.csum = !!dev->data->dev_conf.rxmode.hw_ip_checksum; + if (priv->hw_csum_l2tun) + tmpl.csum_l2tun = !!dev->data->dev_conf.rxmode.hw_ip_checksum; /* Enable scattered packets support for this queue if necessary. */ if ((dev->data->dev_conf.rxmode.jumbo_frame) && (dev->data->dev_conf.rxmode.max_rx_pkt_len > @@ -3082,7 +3323,24 @@ rxq_setup(struct rte_eth_dev *dev, struct rxq *rxq, uint16_t desc, goto error; } skip_mr: - tmpl.cq = ibv_create_cq(priv->ctx, desc, NULL, NULL, 0); + attr.rd = (struct ibv_exp_res_domain_init_attr){ + .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | + IBV_EXP_RES_DOMAIN_MSG_MODEL), + .thread_model = IBV_EXP_THREAD_SINGLE, + .msg_model = IBV_EXP_MSG_HIGH_BW, + }; + tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); + if (tmpl.rd == NULL) { + ret = ENOMEM; + ERROR("%p: RD creation failure: %s", + (void *)dev, strerror(ret)); + goto error; + } + attr.cq = (struct ibv_exp_cq_init_attr){ + .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, + .res_domain = tmpl.rd, + }; + tmpl.cq = ibv_exp_create_cq(priv->ctx, desc, NULL, NULL, 0, &attr.cq); if (tmpl.cq == NULL) { ret = ENOMEM; ERROR("%p: CQ creation failure: %s", @@ -3095,10 +3353,11 @@ skip_mr: priv->device_attr.max_sge); #ifdef RSS_SUPPORT if (priv->rss) - tmpl.qp = rxq_setup_qp_rss(priv, tmpl.cq, desc, parent); + tmpl.qp = rxq_setup_qp_rss(priv, tmpl.cq, desc, parent, + tmpl.rd); else #endif /* RSS_SUPPORT */ - tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc); + tmpl.qp = rxq_setup_qp(priv, tmpl.cq, desc, tmpl.rd); if (tmpl.qp == NULL) { ret = (errno ? errno : EINVAL); ERROR("%p: QP creation failure: %s", @@ -3168,6 +3427,28 @@ skip_alloc: /* Save port ID. */ tmpl.port_id = dev->data->port_id; DEBUG("%p: RTE port ID: %u", (void *)rxq, tmpl.port_id); + attr.params = (struct ibv_exp_query_intf_params){ + .intf_scope = IBV_EXP_INTF_GLOBAL, + .intf = IBV_EXP_INTF_CQ, + .obj = tmpl.cq, + }; + tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); + if (tmpl.if_cq == NULL) { + ERROR("%p: CQ interface family query failed with status %d", + (void *)dev, status); + goto error; + } + attr.params = (struct ibv_exp_query_intf_params){ + .intf_scope = IBV_EXP_INTF_GLOBAL, + .intf = IBV_EXP_INTF_QP_BURST, + .obj = tmpl.qp, + }; + tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status); + if (tmpl.if_qp == NULL) { + ERROR("%p: QP interface family query failed with status %d", + (void *)dev, status); + goto error; + } /* Clean up rxq in case we're reinitializing it. */ DEBUG("%p: cleaning-up old rxq just in case", (void *)rxq); rxq_cleanup(rxq); @@ -3535,6 +3816,18 @@ mlx4_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info) info->max_rx_queues = max; info->max_tx_queues = max; info->max_mac_addrs = elemof(priv->mac); + info->rx_offload_capa = + (priv->hw_csum ? + (DEV_RX_OFFLOAD_IPV4_CKSUM | + DEV_RX_OFFLOAD_UDP_CKSUM | + DEV_RX_OFFLOAD_TCP_CKSUM) : + 0); + info->tx_offload_capa = + (priv->hw_csum ? + (DEV_TX_OFFLOAD_IPV4_CKSUM | + DEV_TX_OFFLOAD_UDP_CKSUM | + DEV_TX_OFFLOAD_TCP_CKSUM) : + 0); priv_unlock(priv); } @@ -3863,29 +4156,37 @@ static int mlx4_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete) { struct priv *priv = dev->data->dev_private; - struct ibv_port_attr port_attr; - static const uint8_t width_mult[] = { - /* Multiplier values taken from devinfo.c in libibverbs. */ - 0, 1, 4, 0, 8, 0, 0, 0, 12, 0 + struct ethtool_cmd edata = { + .cmd = ETHTOOL_GSET }; + struct ifreq ifr; + struct rte_eth_link dev_link; + int link_speed = 0; (void)wait_to_complete; - errno = ibv_query_port(priv->ctx, priv->port, &port_attr); - if (errno) { - WARN("port query failed: %s", strerror(errno)); + if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) { + WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno)); return -1; } - dev->data->dev_link = (struct rte_eth_link){ - .link_speed = (ibv_rate_to_mbps(mult_to_ibv_rate - (port_attr.active_speed)) * - width_mult[(port_attr.active_width % - sizeof(width_mult))]), - .link_duplex = ETH_LINK_FULL_DUPLEX, - .link_status = (port_attr.state == IBV_PORT_ACTIVE) - }; - if (memcmp(&port_attr, &priv->port_attr, sizeof(port_attr))) { + memset(&dev_link, 0, sizeof(dev_link)); + dev_link.link_status = ((ifr.ifr_flags & IFF_UP) && + (ifr.ifr_flags & IFF_RUNNING)); + ifr.ifr_data = &edata; + if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) { + WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s", + strerror(errno)); + return -1; + } + link_speed = ethtool_cmd_speed(&edata); + if (link_speed == -1) + dev_link.link_speed = 0; + else + dev_link.link_speed = link_speed; + dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ? + ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX); + if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) { /* Link status changed. */ - priv->port_attr = port_attr; + dev->data->dev_link = dev_link; return 0; } /* Link status is still the same. */ @@ -4245,6 +4546,8 @@ static const struct eth_dev_ops mlx4_dev_ops = { .mac_addr_remove = mlx4_mac_addr_remove, .mac_addr_add = mlx4_mac_addr_add, .mtu_set = mlx4_dev_set_mtu, + .udp_tunnel_add = NULL, + .udp_tunnel_del = NULL, .fdir_add_signature_filter = NULL, .fdir_update_signature_filter = NULL, .fdir_remove_signature_filter = NULL, @@ -4305,22 +4608,25 @@ mlx4_ibv_device_to_pci_addr(const struct ibv_device *device, } /** - * Derive MAC address from port GID. + * Get MAC address by querying netdevice. * + * @param[in] priv + * struct priv for the requested device. * @param[out] mac * MAC address output buffer. - * @param port - * Physical port number. - * @param[in] gid - * Port GID. + * + * @return + * 0 on success, -1 on failure and errno is set. */ -static void -mac_from_gid(uint8_t (*mac)[ETHER_ADDR_LEN], uint32_t port, uint8_t *gid) +static int +priv_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN]) { - memcpy(&(*mac)[0], gid + 8, 3); - memcpy(&(*mac)[3], gid + 13, 3); - if (port == 1) - (*mac)[0] ^= 2; + struct ifreq request; + + if (priv_ifreq(priv, SIOCGIFHWADDR, &request)) + return -1; + memcpy(mac, request.ifr_hwaddr.sa_data, ETHER_ADDR_LEN); + return 0; } /* Support up to 32 adapters. */ @@ -4482,7 +4788,6 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) struct ibv_exp_device_attr exp_device_attr; #endif /* HAVE_EXP_QUERY_DEVICE */ struct ether_addr mac; - union ibv_gid temp_gid; #ifdef HAVE_EXP_QUERY_DEVICE exp_device_attr.comp_mask = IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS; @@ -4530,7 +4835,6 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) priv->ctx = ctx; priv->device_attr = device_attr; - priv->port_attr = port_attr; priv->port = port; priv->pd = pd; priv->mtu = ETHER_MTU; @@ -4566,6 +4870,19 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) exp_device_attr.max_rss_tbl_sz); #endif /* RSS_SUPPORT */ + priv->hw_csum = + ((exp_device_attr.exp_device_cap_flags & + IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT) && + (exp_device_attr.exp_device_cap_flags & + IBV_EXP_DEVICE_RX_CSUM_IP_PKT)); + DEBUG("checksum offloading is %ssupported", + (priv->hw_csum ? "" : "not ")); + + priv->hw_csum_l2tun = !!(exp_device_attr.exp_device_cap_flags & + IBV_EXP_DEVICE_VXLAN_SUPPORT); + DEBUG("L2 tunnel checksum offloads are %ssupported", + (priv->hw_csum_l2tun ? "" : "not ")); + #ifdef INLINE_RECV priv->inl_recv_size = mlx4_getenv_int("MLX4_INLINE_RECV_SIZE"); @@ -4595,12 +4912,12 @@ mlx4_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev) (void)mlx4_getenv_int; priv->vf = vf; - if (ibv_query_gid(ctx, port, 0, &temp_gid)) { - ERROR("ibv_query_gid() failure"); + /* Configure the first MAC address by default. */ + if (priv_get_mac(priv, &mac.addr_bytes)) { + ERROR("cannot get MAC address, is mlx4_en loaded?" + " (errno: %s)", strerror(errno)); goto port_error; } - /* Configure the first MAC address by default. */ - mac_from_gid(&mac.addr_bytes, port, temp_gid.raw); INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x", priv->port, mac.addr_bytes[0], mac.addr_bytes[1], @@ -4729,6 +5046,14 @@ rte_mlx4_pmd_init(const char *name, const char *args) { (void)name; (void)args; + /* + * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use + * huge pages. Calling ibv_fork_init() during init allows + * applications to use fork() safely for purposes other than + * using this PMD, which is not supported in forked processes. + */ + setenv("RDMAV_HUGEPAGES_SAFE", "1", 1); + ibv_fork_init(); rte_eal_pci_register(&mlx4_driver.pci_drv); return 0; }