X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_rxq.c;h=e7ec1dae329b3318b7f4257077fe8774e4a447ae;hb=a4a4e3fd8f4473b40c0fca6394144e499b0d76c0;hp=c603d2b0a8f46eecf30a82f44c01cda29dcc8af3;hpb=272733b5ebfd1ec44dc801c36cfbe4af761b4bd7;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c index c603d2b0a8..e7ec1dae32 100644 --- a/drivers/net/mlx5/mlx5_rxq.c +++ b/drivers/net/mlx5/mlx5_rxq.c @@ -64,122 +64,6 @@ #include "mlx5_autoconf.h" #include "mlx5_defs.h" -/* Initialization data for hash RX queues. */ -const struct hash_rxq_init hash_rxq_init[] = { - [HASH_RXQ_TCPV4] = { - .hash_fields = (IBV_RX_HASH_SRC_IPV4 | - IBV_RX_HASH_DST_IPV4 | - IBV_RX_HASH_SRC_PORT_TCP | - IBV_RX_HASH_DST_PORT_TCP), - .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_TCP, - .flow_priority = 0, - .flow_spec.tcp_udp = { - .type = IBV_FLOW_SPEC_TCP, - .size = sizeof(hash_rxq_init[0].flow_spec.tcp_udp), - }, - .underlayer = &hash_rxq_init[HASH_RXQ_IPV4], - }, - [HASH_RXQ_UDPV4] = { - .hash_fields = (IBV_RX_HASH_SRC_IPV4 | - IBV_RX_HASH_DST_IPV4 | - IBV_RX_HASH_SRC_PORT_UDP | - IBV_RX_HASH_DST_PORT_UDP), - .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV4_UDP, - .flow_priority = 0, - .flow_spec.tcp_udp = { - .type = IBV_FLOW_SPEC_UDP, - .size = sizeof(hash_rxq_init[0].flow_spec.tcp_udp), - }, - .underlayer = &hash_rxq_init[HASH_RXQ_IPV4], - }, - [HASH_RXQ_IPV4] = { - .hash_fields = (IBV_RX_HASH_SRC_IPV4 | - IBV_RX_HASH_DST_IPV4), - .dpdk_rss_hf = (ETH_RSS_IPV4 | - ETH_RSS_FRAG_IPV4), - .flow_priority = 1, - .flow_spec.ipv4 = { - .type = IBV_FLOW_SPEC_IPV4, - .size = sizeof(hash_rxq_init[0].flow_spec.ipv4), - }, - .underlayer = &hash_rxq_init[HASH_RXQ_ETH], - }, - [HASH_RXQ_TCPV6] = { - .hash_fields = (IBV_RX_HASH_SRC_IPV6 | - IBV_RX_HASH_DST_IPV6 | - IBV_RX_HASH_SRC_PORT_TCP | - IBV_RX_HASH_DST_PORT_TCP), - .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_TCP, - .flow_priority = 0, - .flow_spec.tcp_udp = { - .type = IBV_FLOW_SPEC_TCP, - .size = sizeof(hash_rxq_init[0].flow_spec.tcp_udp), - }, - .underlayer = &hash_rxq_init[HASH_RXQ_IPV6], - }, - [HASH_RXQ_UDPV6] = { - .hash_fields = (IBV_RX_HASH_SRC_IPV6 | - IBV_RX_HASH_DST_IPV6 | - IBV_RX_HASH_SRC_PORT_UDP | - IBV_RX_HASH_DST_PORT_UDP), - .dpdk_rss_hf = ETH_RSS_NONFRAG_IPV6_UDP, - .flow_priority = 0, - .flow_spec.tcp_udp = { - .type = IBV_FLOW_SPEC_UDP, - .size = sizeof(hash_rxq_init[0].flow_spec.tcp_udp), - }, - .underlayer = &hash_rxq_init[HASH_RXQ_IPV6], - }, - [HASH_RXQ_IPV6] = { - .hash_fields = (IBV_RX_HASH_SRC_IPV6 | - IBV_RX_HASH_DST_IPV6), - .dpdk_rss_hf = (ETH_RSS_IPV6 | - ETH_RSS_FRAG_IPV6), - .flow_priority = 1, - .flow_spec.ipv6 = { - .type = IBV_FLOW_SPEC_IPV6, - .size = sizeof(hash_rxq_init[0].flow_spec.ipv6), - }, - .underlayer = &hash_rxq_init[HASH_RXQ_ETH], - }, - [HASH_RXQ_ETH] = { - .hash_fields = 0, - .dpdk_rss_hf = 0, - .flow_priority = 2, - .flow_spec.eth = { - .type = IBV_FLOW_SPEC_ETH, - .size = sizeof(hash_rxq_init[0].flow_spec.eth), - }, - .underlayer = NULL, - }, -}; - -/* Number of entries in hash_rxq_init[]. */ -const unsigned int hash_rxq_init_n = RTE_DIM(hash_rxq_init); - -/* Initialization data for hash RX queue indirection tables. */ -static const struct ind_table_init ind_table_init[] = { - { - .max_size = -1u, /* Superseded by HW limitations. */ - .hash_types = - 1 << HASH_RXQ_TCPV4 | - 1 << HASH_RXQ_UDPV4 | - 1 << HASH_RXQ_IPV4 | - 1 << HASH_RXQ_TCPV6 | - 1 << HASH_RXQ_UDPV6 | - 1 << HASH_RXQ_IPV6 | - 0, - .hash_types_n = 6, - }, - { - .max_size = 1, - .hash_types = 1 << HASH_RXQ_ETH, - .hash_types_n = 1, - }, -}; - -#define IND_TABLE_INIT_N RTE_DIM(ind_table_init) - /* Default RSS hash key also used for ConnectX-3. */ uint8_t rss_hash_default_key[] = { 0x2c, 0xc6, 0x81, 0xd1, @@ -197,360 +81,6 @@ uint8_t rss_hash_default_key[] = { /* Length of the default RSS hash key. */ const size_t rss_hash_default_key_len = sizeof(rss_hash_default_key); -/** - * Populate flow steering rule for a given hash RX queue type using - * information from hash_rxq_init[]. Nothing is written to flow_attr when - * flow_attr_size is not large enough, but the required size is still returned. - * - * @param priv - * Pointer to private structure. - * @param[out] flow_attr - * Pointer to flow attribute structure to fill. Note that the allocated - * area must be larger and large enough to hold all flow specifications. - * @param flow_attr_size - * Entire size of flow_attr and trailing room for flow specifications. - * @param type - * Hash RX queue type to use for flow steering rule. - * - * @return - * Total size of the flow attribute buffer. No errors are defined. - */ -size_t -priv_flow_attr(struct priv *priv, struct ibv_flow_attr *flow_attr, - size_t flow_attr_size, enum hash_rxq_type type) -{ - size_t offset = sizeof(*flow_attr); - const struct hash_rxq_init *init = &hash_rxq_init[type]; - - assert(priv != NULL); - assert((size_t)type < RTE_DIM(hash_rxq_init)); - do { - offset += init->flow_spec.hdr.size; - init = init->underlayer; - } while (init != NULL); - if (offset > flow_attr_size) - return offset; - flow_attr_size = offset; - init = &hash_rxq_init[type]; - *flow_attr = (struct ibv_flow_attr){ - .type = IBV_FLOW_ATTR_NORMAL, - /* Priorities < 3 are reserved for flow director. */ - .priority = init->flow_priority + 3, - .num_of_specs = 0, - .port = priv->port, - .flags = 0, - }; - do { - offset -= init->flow_spec.hdr.size; - memcpy((void *)((uintptr_t)flow_attr + offset), - &init->flow_spec, - init->flow_spec.hdr.size); - ++flow_attr->num_of_specs; - init = init->underlayer; - } while (init != NULL); - return flow_attr_size; -} - -/** - * Convert hash type position in indirection table initializer to - * hash RX queue type. - * - * @param table - * Indirection table initializer. - * @param pos - * Hash type position. - * - * @return - * Hash RX queue type. - */ -static enum hash_rxq_type -hash_rxq_type_from_pos(const struct ind_table_init *table, unsigned int pos) -{ - enum hash_rxq_type type = HASH_RXQ_TCPV4; - - assert(pos < table->hash_types_n); - do { - if ((table->hash_types & (1 << type)) && (pos-- == 0)) - break; - ++type; - } while (1); - return type; -} - -/** - * Filter out disabled hash RX queue types from ind_table_init[]. - * - * @param priv - * Pointer to private structure. - * @param[out] table - * Output table. - * - * @return - * Number of table entries. - */ -static unsigned int -priv_make_ind_table_init(struct priv *priv, - struct ind_table_init (*table)[IND_TABLE_INIT_N]) -{ - uint64_t rss_hf; - unsigned int i; - unsigned int j; - unsigned int table_n = 0; - /* Mandatory to receive frames not handled by normal hash RX queues. */ - unsigned int hash_types_sup = 1 << HASH_RXQ_ETH; - - rss_hf = priv->rss_hf; - /* Process other protocols only if more than one queue. */ - if (priv->rxqs_n > 1) - for (i = 0; (i != hash_rxq_init_n); ++i) - if (rss_hf & hash_rxq_init[i].dpdk_rss_hf) - hash_types_sup |= (1 << i); - - /* Filter out entries whose protocols are not in the set. */ - for (i = 0, j = 0; (i != IND_TABLE_INIT_N); ++i) { - unsigned int nb; - unsigned int h; - - /* j is increased only if the table has valid protocols. */ - assert(j <= i); - (*table)[j] = ind_table_init[i]; - (*table)[j].hash_types &= hash_types_sup; - for (h = 0, nb = 0; (h != hash_rxq_init_n); ++h) - if (((*table)[j].hash_types >> h) & 0x1) - ++nb; - (*table)[i].hash_types_n = nb; - if (nb) { - ++table_n; - ++j; - } - } - return table_n; -} - -/** - * Initialize hash RX queues and indirection table. - * - * @param priv - * Pointer to private structure. - * - * @return - * 0 on success, errno value on failure. - */ -int -priv_create_hash_rxqs(struct priv *priv) -{ - struct ibv_wq *wqs[priv->reta_idx_n]; - struct ind_table_init ind_table_init[IND_TABLE_INIT_N]; - unsigned int ind_tables_n = - priv_make_ind_table_init(priv, &ind_table_init); - unsigned int hash_rxqs_n = 0; - struct hash_rxq (*hash_rxqs)[] = NULL; - struct ibv_rwq_ind_table *(*ind_tables)[] = NULL; - unsigned int i; - unsigned int j; - unsigned int k; - int err = 0; - - assert(priv->ind_tables == NULL); - assert(priv->ind_tables_n == 0); - assert(priv->hash_rxqs == NULL); - assert(priv->hash_rxqs_n == 0); - assert(priv->pd != NULL); - assert(priv->ctx != NULL); - if (priv->isolated) - return 0; - if (priv->rxqs_n == 0) - return EINVAL; - assert(priv->rxqs != NULL); - if (ind_tables_n == 0) { - ERROR("all hash RX queue types have been filtered out," - " indirection table cannot be created"); - return EINVAL; - } - if (priv->rxqs_n & (priv->rxqs_n - 1)) { - INFO("%u RX queues are configured, consider rounding this" - " number to the next power of two for better balancing", - priv->rxqs_n); - DEBUG("indirection table extended to assume %u WQs", - priv->reta_idx_n); - } - for (i = 0; (i != priv->reta_idx_n); ++i) { - struct mlx5_rxq_ctrl *rxq_ctrl; - - rxq_ctrl = container_of((*priv->rxqs)[(*priv->reta_idx)[i]], - struct mlx5_rxq_ctrl, rxq); - wqs[i] = rxq_ctrl->ibv->wq; - } - /* Get number of hash RX queues to configure. */ - for (i = 0, hash_rxqs_n = 0; (i != ind_tables_n); ++i) - hash_rxqs_n += ind_table_init[i].hash_types_n; - DEBUG("allocating %u hash RX queues for %u WQs, %u indirection tables", - hash_rxqs_n, priv->rxqs_n, ind_tables_n); - /* Create indirection tables. */ - ind_tables = rte_calloc(__func__, ind_tables_n, - sizeof((*ind_tables)[0]), 0); - if (ind_tables == NULL) { - err = ENOMEM; - ERROR("cannot allocate indirection tables container: %s", - strerror(err)); - goto error; - } - for (i = 0; (i != ind_tables_n); ++i) { - struct ibv_rwq_ind_table_init_attr ind_init_attr = { - .log_ind_tbl_size = 0, /* Set below. */ - .ind_tbl = wqs, - .comp_mask = 0, - }; - unsigned int ind_tbl_size = ind_table_init[i].max_size; - struct ibv_rwq_ind_table *ind_table; - - if (priv->reta_idx_n < ind_tbl_size) - ind_tbl_size = priv->reta_idx_n; - ind_init_attr.log_ind_tbl_size = log2above(ind_tbl_size); - errno = 0; - ind_table = ibv_create_rwq_ind_table(priv->ctx, - &ind_init_attr); - if (ind_table != NULL) { - (*ind_tables)[i] = ind_table; - continue; - } - /* Not clear whether errno is set. */ - err = (errno ? errno : EINVAL); - ERROR("RX indirection table creation failed with error %d: %s", - err, strerror(err)); - goto error; - } - /* Allocate array that holds hash RX queues and related data. */ - hash_rxqs = rte_calloc(__func__, hash_rxqs_n, - sizeof((*hash_rxqs)[0]), 0); - if (hash_rxqs == NULL) { - err = ENOMEM; - ERROR("cannot allocate hash RX queues container: %s", - strerror(err)); - goto error; - } - for (i = 0, j = 0, k = 0; - ((i != hash_rxqs_n) && (j != ind_tables_n)); - ++i) { - struct hash_rxq *hash_rxq = &(*hash_rxqs)[i]; - enum hash_rxq_type type = - hash_rxq_type_from_pos(&ind_table_init[j], k); - struct rte_eth_rss_conf *priv_rss_conf = - (*priv->rss_conf)[type]; - struct ibv_rx_hash_conf hash_conf = { - .rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ, - .rx_hash_key_len = (priv_rss_conf ? - priv_rss_conf->rss_key_len : - rss_hash_default_key_len), - .rx_hash_key = (priv_rss_conf ? - priv_rss_conf->rss_key : - rss_hash_default_key), - .rx_hash_fields_mask = hash_rxq_init[type].hash_fields, - }; - struct ibv_qp_init_attr_ex qp_init_attr = { - .qp_type = IBV_QPT_RAW_PACKET, - .comp_mask = (IBV_QP_INIT_ATTR_PD | - IBV_QP_INIT_ATTR_IND_TABLE | - IBV_QP_INIT_ATTR_RX_HASH), - .rx_hash_conf = hash_conf, - .rwq_ind_tbl = (*ind_tables)[j], - .pd = priv->pd, - }; - - DEBUG("using indirection table %u for hash RX queue %u type %d", - j, i, type); - *hash_rxq = (struct hash_rxq){ - .priv = priv, - .qp = ibv_create_qp_ex(priv->ctx, &qp_init_attr), - .type = type, - }; - if (hash_rxq->qp == NULL) { - err = (errno ? errno : EINVAL); - ERROR("Hash RX QP creation failure: %s", - strerror(err)); - goto error; - } - if (++k < ind_table_init[j].hash_types_n) - continue; - /* Switch to the next indirection table and reset hash RX - * queue type array index. */ - ++j; - k = 0; - } - priv->ind_tables = ind_tables; - priv->ind_tables_n = ind_tables_n; - priv->hash_rxqs = hash_rxqs; - priv->hash_rxqs_n = hash_rxqs_n; - assert(err == 0); - return 0; -error: - if (hash_rxqs != NULL) { - for (i = 0; (i != hash_rxqs_n); ++i) { - struct ibv_qp *qp = (*hash_rxqs)[i].qp; - - if (qp == NULL) - continue; - claim_zero(ibv_destroy_qp(qp)); - } - rte_free(hash_rxqs); - } - if (ind_tables != NULL) { - for (j = 0; (j != ind_tables_n); ++j) { - struct ibv_rwq_ind_table *ind_table = - (*ind_tables)[j]; - - if (ind_table == NULL) - continue; - claim_zero(ibv_destroy_rwq_ind_table(ind_table)); - } - rte_free(ind_tables); - } - return err; -} - -/** - * Clean up hash RX queues and indirection table. - * - * @param priv - * Pointer to private structure. - */ -void -priv_destroy_hash_rxqs(struct priv *priv) -{ - unsigned int i; - - DEBUG("destroying %u hash RX queues", priv->hash_rxqs_n); - if (priv->hash_rxqs_n == 0) { - assert(priv->hash_rxqs == NULL); - assert(priv->ind_tables == NULL); - return; - } - for (i = 0; (i != priv->hash_rxqs_n); ++i) { - struct hash_rxq *hash_rxq = &(*priv->hash_rxqs)[i]; - unsigned int j, k; - - assert(hash_rxq->priv == priv); - assert(hash_rxq->qp != NULL); - for (j = 0; (j != RTE_DIM(hash_rxq->mac_flow)); ++j) - for (k = 0; (k != RTE_DIM(hash_rxq->mac_flow[j])); ++k) - assert(hash_rxq->mac_flow[j][k] == NULL); - claim_zero(ibv_destroy_qp(hash_rxq->qp)); - } - priv->hash_rxqs_n = 0; - rte_free(priv->hash_rxqs); - priv->hash_rxqs = NULL; - for (i = 0; (i != priv->ind_tables_n); ++i) { - struct ibv_rwq_ind_table *ind_table = - (*priv->ind_tables)[i]; - - assert(ind_table != NULL); - claim_zero(ibv_destroy_rwq_ind_table(ind_table)); - } - priv->ind_tables_n = 0; - rte_free(priv->ind_tables); - priv->ind_tables = NULL; -} - /** * Allocate RX queue elements. *