X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_rxq.c;h=8b7823360c9ecf45c83f8cd250ccadc7d1da92a6;hb=3b57c21ef9d0c434968798338292d14ab61290d4;hp=908fd0f2fe7d8547b34f7359954bb26ec4fc5251;hpb=9964b965ad695d596130c05ea602c2920f7f9887;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index 908fd0f2fe..8b7823360c 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -36,29 +36,32 @@ #include #include #include +#include /* Verbs header. */ /* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include #include #include #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif /* DPDK headers don't like -pedantic. */ #ifdef PEDANTIC -#pragma GCC diagnostic ignored "-pedantic" +#pragma GCC diagnostic ignored "-Wpedantic" #endif #include #include #include #include +#include +#include #ifdef PEDANTIC -#pragma GCC diagnostic error "-pedantic" +#pragma GCC diagnostic error "-Wpedantic" #endif #include "mlx5.h" @@ -269,7 +272,7 @@ priv_flow_attr(struct priv *priv, struct ibv_exp_flow_attr *flow_attr, static enum hash_rxq_type hash_rxq_type_from_pos(const struct ind_table_init *table, unsigned int pos) { - enum hash_rxq_type type = 0; + enum hash_rxq_type type = HASH_RXQ_TCPV4; assert(pos < table->hash_types_n); do { @@ -609,9 +612,11 @@ priv_allow_flow_type(struct priv *priv, enum hash_rxq_flow_type type) int priv_rehash_flows(struct priv *priv) { - unsigned int i; + enum hash_rxq_flow_type i; - for (i = 0; (i != RTE_DIM((*priv->hash_rxqs)[0].special_flow)); ++i) + for (i = HASH_RXQ_FLOW_TYPE_PROMISC; + i != RTE_DIM((*priv->hash_rxqs)[0].special_flow); + ++i) if (!priv_allow_flow_type(priv, i)) { priv_special_flow_disable(priv, i); } else { @@ -721,7 +726,7 @@ rxq_free_elts(struct rxq_ctrl *rxq_ctrl) if (rxq_ctrl->rxq.elts == NULL) return; - for (i = 0; (i != rxq_ctrl->rxq.elts_n); ++i) { + for (i = 0; (i != (1u << rxq_ctrl->rxq.elts_n)); ++i) { if ((*rxq_ctrl->rxq.elts)[i] != NULL) rte_pktmbuf_free_seg((*rxq_ctrl->rxq.elts)[i]); (*rxq_ctrl->rxq.elts)[i] = NULL; @@ -739,47 +744,16 @@ rxq_free_elts(struct rxq_ctrl *rxq_ctrl) void rxq_cleanup(struct rxq_ctrl *rxq_ctrl) { - struct ibv_exp_release_intf_params params; - DEBUG("cleaning up %p", (void *)rxq_ctrl); rxq_free_elts(rxq_ctrl); - if (rxq_ctrl->if_wq != NULL) { - assert(rxq_ctrl->priv != NULL); - assert(rxq_ctrl->priv->ctx != NULL); - assert(rxq_ctrl->wq != NULL); - params = (struct ibv_exp_release_intf_params){ - .comp_mask = 0, - }; - claim_zero(ibv_exp_release_intf(rxq_ctrl->priv->ctx, - rxq_ctrl->if_wq, - ¶ms)); - } - if (rxq_ctrl->if_cq != NULL) { - assert(rxq_ctrl->priv != NULL); - assert(rxq_ctrl->priv->ctx != NULL); - assert(rxq_ctrl->cq != NULL); - params = (struct ibv_exp_release_intf_params){ - .comp_mask = 0, - }; - claim_zero(ibv_exp_release_intf(rxq_ctrl->priv->ctx, - rxq_ctrl->if_cq, - ¶ms)); - } + if (rxq_ctrl->fdir_queue != NULL) + priv_fdir_queue_destroy(rxq_ctrl->priv, rxq_ctrl->fdir_queue); if (rxq_ctrl->wq != NULL) claim_zero(ibv_exp_destroy_wq(rxq_ctrl->wq)); if (rxq_ctrl->cq != NULL) claim_zero(ibv_destroy_cq(rxq_ctrl->cq)); - if (rxq_ctrl->rd != NULL) { - struct ibv_exp_destroy_res_domain_attr attr = { - .comp_mask = 0, - }; - - assert(rxq_ctrl->priv != NULL); - assert(rxq_ctrl->priv->ctx != NULL); - claim_zero(ibv_exp_destroy_res_domain(rxq_ctrl->priv->ctx, - rxq_ctrl->rd, - &attr)); - } + if (rxq_ctrl->channel != NULL) + claim_zero(ibv_destroy_comp_channel(rxq_ctrl->channel)); if (rxq_ctrl->mr != NULL) claim_zero(ibv_dereg_mr(rxq_ctrl->mr)); memset(rxq_ctrl, 0, sizeof(*rxq_ctrl)); @@ -803,7 +777,7 @@ rxq_cleanup(struct rxq_ctrl *rxq_ctrl) int rxq_rehash(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl) { - unsigned int elts_n = rxq_ctrl->rxq.elts_n; + unsigned int elts_n = 1 << rxq_ctrl->rxq.elts_n; unsigned int i; struct ibv_exp_wq_attr mod; int err; @@ -866,7 +840,7 @@ rxq_setup(struct rxq_ctrl *tmpl) struct ibv_cq *ibcq = tmpl->cq; struct mlx5_cq *cq = to_mxxx(cq, cq); struct mlx5_rwq *rwq = container_of(tmpl->wq, struct mlx5_rwq, wq); - struct rte_mbuf *(*elts)[tmpl->rxq.elts_n] = + struct rte_mbuf *(*elts)[1 << tmpl->rxq.elts_n] = rte_calloc_socket("RXQ", 1, sizeof(*elts), 0, tmpl->socket); if (cq->cqe_sz != RTE_CACHE_LINE_SIZE) { @@ -877,7 +851,7 @@ rxq_setup(struct rxq_ctrl *tmpl) if (elts == NULL) return ENOMEM; tmpl->rxq.rq_db = rwq->rq.db; - tmpl->rxq.cqe_n = ibcq->cqe + 1; + tmpl->rxq.cqe_n = log2above(ibcq->cqe); tmpl->rxq.cq_ci = 0; tmpl->rxq.rq_ci = 0; tmpl->rxq.cq_db = cq->dbrec; @@ -920,19 +894,17 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, .priv = priv, .socket = socket, .rxq = { - .elts_n = desc, + .elts_n = log2above(desc), .mp = mp, + .rss_hash = priv->rxqs_n > 1, }, }; struct ibv_exp_wq_attr mod; union { - struct ibv_exp_query_intf_params params; struct ibv_exp_cq_init_attr cq; - struct ibv_exp_res_domain_init_attr rd; struct ibv_exp_wq_init_attr wq; struct ibv_exp_cq_attr cq_attr; } attr; - enum ibv_exp_query_intf_status status; unsigned int mb_len = rte_pktmbuf_data_room_size(mp); unsigned int cqe_n = desc - 1; struct rte_mbuf *(*elts)[desc] = NULL; @@ -941,9 +913,10 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, (void)conf; /* Thresholds configuration (ignored). */ /* Enable scattered packets support for this queue if necessary. */ assert(mb_len >= RTE_PKTMBUF_HEADROOM); - if ((dev->data->dev_conf.rxmode.jumbo_frame) && - (dev->data->dev_conf.rxmode.max_rx_pkt_len > - (mb_len - RTE_PKTMBUF_HEADROOM))) { + if (dev->data->dev_conf.rxmode.max_rx_pkt_len <= + (mb_len - RTE_PKTMBUF_HEADROOM)) { + tmpl.rxq.sges_n = 0; + } else if (dev->data->dev_conf.rxmode.enable_scatter) { unsigned int size = RTE_PKTMBUF_HEADROOM + dev->data->dev_conf.rxmode.max_rx_pkt_len; @@ -966,6 +939,13 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, dev->data->dev_conf.rxmode.max_rx_pkt_len); return EOVERFLOW; } + } else { + WARN("%p: the requested maximum Rx packet size (%u) is" + " larger than a single mbuf (%u) and scattered" + " mode has not been requested", + (void *)dev, + dev->data->dev_conf.rxmode.max_rx_pkt_len, + mb_len - RTE_PKTMBUF_HEADROOM); } DEBUG("%p: maximum number of segments per packet: %u", (void *)dev, 1 << tmpl.rxq.sges_n); @@ -991,29 +971,25 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, (void *)dev, strerror(ret)); goto error; } - attr.rd = (struct ibv_exp_res_domain_init_attr){ - .comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL | - IBV_EXP_RES_DOMAIN_MSG_MODEL), - .thread_model = IBV_EXP_THREAD_SINGLE, - .msg_model = IBV_EXP_MSG_HIGH_BW, - }; - tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd); - if (tmpl.rd == NULL) { - ret = ENOMEM; - ERROR("%p: RD creation failure: %s", - (void *)dev, strerror(ret)); - goto error; + if (dev->data->dev_conf.intr_conf.rxq) { + tmpl.channel = ibv_create_comp_channel(priv->ctx); + if (tmpl.channel == NULL) { + dev->data->dev_conf.intr_conf.rxq = 0; + ret = ENOMEM; + ERROR("%p: Comp Channel creation failure: %s", + (void *)dev, strerror(ret)); + goto error; + } } attr.cq = (struct ibv_exp_cq_init_attr){ - .comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN, - .res_domain = tmpl.rd, + .comp_mask = 0, }; if (priv->cqe_comp) { attr.cq.comp_mask |= IBV_EXP_CQ_INIT_ATTR_FLAGS; attr.cq.flags |= IBV_EXP_CQ_COMPRESSED_CQE; cqe_n = (desc * 2) - 1; /* Double the number of CQEs. */ } - tmpl.cq = ibv_exp_create_cq(priv->ctx, cqe_n, NULL, NULL, 0, + tmpl.cq = ibv_exp_create_cq(priv->ctx, cqe_n, NULL, tmpl.channel, 0, &attr.cq); if (tmpl.cq == NULL) { ret = ENOMEM; @@ -1038,10 +1014,8 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, .pd = priv->pd, .cq = tmpl.cq, .comp_mask = - IBV_EXP_CREATE_WQ_RES_DOMAIN | IBV_EXP_CREATE_WQ_VLAN_OFFLOADS | 0, - .res_domain = tmpl.rd, .vlan_offloads = (tmpl.rxq.vlan_strip ? IBV_EXP_RECEIVE_WQ_CVLAN_STRIP : 0), @@ -1102,29 +1076,6 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, /* Save port ID. */ tmpl.rxq.port_id = dev->data->port_id; DEBUG("%p: RTE port ID: %u", (void *)rxq_ctrl, tmpl.rxq.port_id); - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf_version = 1, - .intf = IBV_EXP_INTF_CQ, - .obj = tmpl.cq, - }; - tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_cq == NULL) { - ERROR("%p: CQ interface family query failed with status %d", - (void *)dev, status); - goto error; - } - attr.params = (struct ibv_exp_query_intf_params){ - .intf_scope = IBV_EXP_INTF_GLOBAL, - .intf = IBV_EXP_INTF_WQ, - .obj = tmpl.wq, - }; - tmpl.if_wq = ibv_exp_query_intf(priv->ctx, &attr.params, &status); - if (tmpl.if_wq == NULL) { - ERROR("%p: WQ interface family query failed with status %d", - (void *)dev, status); - goto error; - } /* Change queue state to ready. */ mod = (struct ibv_exp_wq_attr){ .attr_mask = IBV_EXP_WQ_ATTR_STATE, @@ -1144,7 +1095,7 @@ rxq_ctrl_setup(struct rte_eth_dev *dev, struct rxq_ctrl *rxq_ctrl, } /* Reuse buffers from original queue if possible. */ if (rxq_ctrl->rxq.elts_n) { - assert(rxq_ctrl->rxq.elts_n == desc); + assert(1 << rxq_ctrl->rxq.elts_n == desc); assert(rxq_ctrl->rxq.elts != tmpl.rxq.elts); ret = rxq_alloc_elts(&tmpl, desc, rxq_ctrl->rxq.elts); } else @@ -1237,6 +1188,19 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, } (*priv->rxqs)[idx] = NULL; rxq_cleanup(rxq_ctrl); + /* Resize if rxq size is changed. */ + if (rxq_ctrl->rxq.elts_n != log2above(desc)) { + rxq_ctrl = rte_realloc(rxq_ctrl, + sizeof(*rxq_ctrl) + + desc * sizeof(struct rte_mbuf *), + RTE_CACHE_LINE_SIZE); + if (!rxq_ctrl) { + ERROR("%p: unable to reallocate queue index %u", + (void *)dev, idx); + priv_unlock(priv); + return -ENOMEM; + } + } } else { rxq_ctrl = rte_calloc_socket("RXQ", 1, sizeof(*rxq_ctrl) + desc * sizeof(struct rte_mbuf *), @@ -1257,7 +1221,7 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc, (void *)dev, (void *)rxq_ctrl); (*priv->rxqs)[idx] = &rxq_ctrl->rxq; /* Update receive callback. */ - dev->rx_pkt_burst = mlx5_rx_burst; + priv_select_rx_function(priv); } priv_unlock(priv); return -ret; @@ -1285,6 +1249,9 @@ mlx5_rx_queue_release(void *dpdk_rxq) rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq); priv = rxq_ctrl->priv; priv_lock(priv); + if (priv_flow_rxq_in_use(priv, rxq)) + rte_panic("Rx queue %p is still used by a flow and cannot be" + " removed\n", (void *)rxq_ctrl); for (i = 0; (i != priv->rxqs_n); ++i) if ((*priv->rxqs)[i] == rxq) { DEBUG("%p: removing RX queue %p from list", @@ -1337,3 +1304,113 @@ mlx5_rx_burst_secondary_setup(void *dpdk_rxq, struct rte_mbuf **pkts, rxq = (*priv->rxqs)[index]; return priv->dev->rx_pkt_burst(rxq, pkts, pkts_n); } + +/** + * Fill epoll fd list for rxq interrupts. + * + * @param priv + * Private structure. + * + * @return + * 0 on success, negative on failure. + */ +int +priv_intr_efd_enable(struct priv *priv) +{ + unsigned int i; + unsigned int rxqs_n = priv->rxqs_n; + unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); + struct rte_intr_handle *intr_handle = priv->dev->intr_handle; + + if (n == 0) + return 0; + if (n < rxqs_n) { + WARN("rxqs num is larger than EAL max interrupt vector " + "%u > %u unable to supprt rxq interrupts", + rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID); + return -EINVAL; + } + intr_handle->type = RTE_INTR_HANDLE_EXT; + for (i = 0; i != n; ++i) { + struct rxq *rxq = (*priv->rxqs)[i]; + struct rxq_ctrl *rxq_ctrl = + container_of(rxq, struct rxq_ctrl, rxq); + int fd = rxq_ctrl->channel->fd; + int flags; + int rc; + + flags = fcntl(fd, F_GETFL); + rc = fcntl(fd, F_SETFL, flags | O_NONBLOCK); + if (rc < 0) { + WARN("failed to change rxq interrupt file " + "descriptor %d for queue index %d", fd, i); + return -1; + } + intr_handle->efds[i] = fd; + } + intr_handle->nb_efd = n; + return 0; +} + +/** + * Clean epoll fd list for rxq interrupts. + * + * @param priv + * Private structure. + */ +void +priv_intr_efd_disable(struct priv *priv) +{ + struct rte_intr_handle *intr_handle = priv->dev->intr_handle; + + rte_intr_free_epoll_fd(intr_handle); +} + +/** + * Create and init interrupt vector array. + * + * @param priv + * Private structure. + * + * @return + * 0 on success, negative on failure. + */ +int +priv_create_intr_vec(struct priv *priv) +{ + unsigned int rxqs_n = priv->rxqs_n; + unsigned int i; + struct rte_intr_handle *intr_handle = priv->dev->intr_handle; + + if (rxqs_n == 0) + return 0; + intr_handle->intr_vec = (int *) + rte_malloc("intr_vec", rxqs_n * sizeof(int), 0); + if (intr_handle->intr_vec == NULL) { + WARN("Failed to allocate memory for intr_vec " + "rxq interrupt will not be supported"); + return -ENOMEM; + } + for (i = 0; i != rxqs_n; ++i) { + /* 1:1 mapping between rxq and interrupt. */ + intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + i; + } + return 0; +} + +/** + * Destroy init interrupt vector array. + * + * @param priv + * Private structure. + * + * @return + * 0 on success, negative on failure. + */ +void +priv_destroy_intr_vec(struct priv *priv) +{ + struct rte_intr_handle *intr_handle = priv->dev->intr_handle; + + rte_free(intr_handle->intr_vec); +}