1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2020 Amazon.com, Inc. or its affiliates.
6 #include "ena_ethdev.h"
9 #include <ena_admin_defs.h>
11 #define TEST_BIT(val, bit_shift) ((val) & (1UL << (bit_shift)))
13 #define ENA_HF_RSS_ALL_L2 (ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA)
14 #define ENA_HF_RSS_ALL_L3 (ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA)
15 #define ENA_HF_RSS_ALL_L4 (ENA_ADMIN_RSS_L4_SP | ENA_ADMIN_RSS_L4_DP)
16 #define ENA_HF_RSS_ALL_L3_L4 (ENA_HF_RSS_ALL_L3 | ENA_HF_RSS_ALL_L4)
17 #define ENA_HF_RSS_ALL_L2_L3_L4 (ENA_HF_RSS_ALL_L2 | ENA_HF_RSS_ALL_L3_L4)
19 enum ena_rss_hash_fields {
20 ENA_HF_RSS_TCP4 = ENA_HF_RSS_ALL_L3_L4,
21 ENA_HF_RSS_UDP4 = ENA_HF_RSS_ALL_L3_L4,
22 ENA_HF_RSS_TCP6 = ENA_HF_RSS_ALL_L3_L4,
23 ENA_HF_RSS_UDP6 = ENA_HF_RSS_ALL_L3_L4,
24 ENA_HF_RSS_IP4 = ENA_HF_RSS_ALL_L3,
25 ENA_HF_RSS_IP6 = ENA_HF_RSS_ALL_L3,
26 ENA_HF_RSS_IP4_FRAG = ENA_HF_RSS_ALL_L3,
27 ENA_HF_RSS_NOT_IP = ENA_HF_RSS_ALL_L2,
28 ENA_HF_RSS_TCP6_EX = ENA_HF_RSS_ALL_L3_L4,
29 ENA_HF_RSS_IP6_EX = ENA_HF_RSS_ALL_L3,
32 static int ena_fill_indirect_table_default(struct ena_com_dev *ena_dev,
35 static uint64_t ena_admin_hf_to_eth_hf(enum ena_admin_flow_hash_proto proto,
37 static uint16_t ena_eth_hf_to_admin_hf(enum ena_admin_flow_hash_proto proto,
39 static int ena_set_hash_fields(struct ena_com_dev *ena_dev, uint64_t rss_hf);
40 static int ena_rss_hash_set(struct ena_com_dev *ena_dev,
41 struct rte_eth_rss_conf *rss_conf,
42 bool default_allowed);
43 static void ena_reorder_rss_hash_key(uint8_t *reordered_key,
46 static int ena_get_rss_hash_key(struct ena_com_dev *ena_dev, uint8_t *rss_key);
48 void ena_rss_key_fill(void *key, size_t size)
50 static bool key_generated;
51 static uint8_t default_key[ENA_HASH_KEY_SIZE];
54 RTE_ASSERT(size <= ENA_HASH_KEY_SIZE);
57 for (i = 0; i < ENA_HASH_KEY_SIZE; ++i)
58 default_key[i] = rte_rand() & 0xff;
62 rte_memcpy(key, default_key, size);
65 int ena_rss_reta_update(struct rte_eth_dev *dev,
66 struct rte_eth_rss_reta_entry64 *reta_conf,
69 struct ena_adapter *adapter = dev->data->dev_private;
70 struct ena_com_dev *ena_dev = &adapter->ena_dev;
76 if (reta_size == 0 || reta_conf == NULL)
79 if (!(dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_RSS_HASH)) {
81 "RSS was not configured for the PMD\n");
85 if (reta_size > ENA_RX_RSS_TABLE_SIZE) {
87 "Requested indirection table size (%d) is bigger than supported: %d\n",
88 reta_size, ENA_RX_RSS_TABLE_SIZE);
92 for (i = 0 ; i < reta_size ; i++) {
93 /* Each reta_conf is for 64 entries.
94 * To support 128 we use 2 conf of 64.
96 conf_idx = i / RTE_ETH_RETA_GROUP_SIZE;
97 idx = i % RTE_ETH_RETA_GROUP_SIZE;
98 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) {
100 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]);
102 rc = ena_com_indirect_table_fill_entry(ena_dev, i,
104 if (unlikely(rc != 0)) {
106 "Cannot fill indirection table\n");
112 rte_spinlock_lock(&adapter->admin_lock);
113 rc = ena_com_indirect_table_set(ena_dev);
114 rte_spinlock_unlock(&adapter->admin_lock);
115 if (unlikely(rc != 0)) {
116 PMD_DRV_LOG(ERR, "Cannot set the indirection table\n");
120 PMD_DRV_LOG(DEBUG, "RSS configured %d entries for port %d\n",
121 reta_size, dev->data->port_id);
126 /* Query redirection table. */
127 int ena_rss_reta_query(struct rte_eth_dev *dev,
128 struct rte_eth_rss_reta_entry64 *reta_conf,
131 uint32_t indirect_table[ENA_RX_RSS_TABLE_SIZE] = { 0 };
132 struct ena_adapter *adapter = dev->data->dev_private;
133 struct ena_com_dev *ena_dev = &adapter->ena_dev;
139 if (reta_size == 0 || reta_conf == NULL)
142 if (!(dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_RSS_HASH)) {
144 "RSS was not configured for the PMD\n");
148 rte_spinlock_lock(&adapter->admin_lock);
149 rc = ena_com_indirect_table_get(ena_dev, indirect_table);
150 rte_spinlock_unlock(&adapter->admin_lock);
151 if (unlikely(rc != 0)) {
152 PMD_DRV_LOG(ERR, "Cannot get indirection table\n");
156 for (i = 0 ; i < reta_size ; i++) {
157 reta_conf_idx = i / RTE_ETH_RETA_GROUP_SIZE;
158 reta_idx = i % RTE_ETH_RETA_GROUP_SIZE;
159 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx))
160 reta_conf[reta_conf_idx].reta[reta_idx] =
161 ENA_IO_RXQ_IDX_REV(indirect_table[i]);
167 static int ena_fill_indirect_table_default(struct ena_com_dev *ena_dev,
175 for (i = 0; i < tbl_size; ++i) {
177 rc = ena_com_indirect_table_fill_entry(ena_dev, i,
178 ENA_IO_RXQ_IDX(val));
179 if (unlikely(rc != 0)) {
181 "Failed to set %zu indirection table entry with val %" PRIu16 "\n",
190 static uint64_t ena_admin_hf_to_eth_hf(enum ena_admin_flow_hash_proto proto,
195 /* If no fields are activated, then RSS is disabled for this proto */
196 if ((fields & ENA_HF_RSS_ALL_L2_L3_L4) == 0)
199 /* Convert proto to ETH flag */
201 case ENA_ADMIN_RSS_TCP4:
202 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP;
204 case ENA_ADMIN_RSS_UDP4:
205 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP;
207 case ENA_ADMIN_RSS_TCP6:
208 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP;
210 case ENA_ADMIN_RSS_UDP6:
211 rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP;
213 case ENA_ADMIN_RSS_IP4:
214 rss_hf |= RTE_ETH_RSS_IPV4;
216 case ENA_ADMIN_RSS_IP6:
217 rss_hf |= RTE_ETH_RSS_IPV6;
219 case ENA_ADMIN_RSS_IP4_FRAG:
220 rss_hf |= RTE_ETH_RSS_FRAG_IPV4;
222 case ENA_ADMIN_RSS_NOT_IP:
223 rss_hf |= RTE_ETH_RSS_L2_PAYLOAD;
225 case ENA_ADMIN_RSS_TCP6_EX:
226 rss_hf |= RTE_ETH_RSS_IPV6_TCP_EX;
228 case ENA_ADMIN_RSS_IP6_EX:
229 rss_hf |= RTE_ETH_RSS_IPV6_EX;
235 /* Check if only DA or SA is being used for L3. */
236 switch (fields & ENA_HF_RSS_ALL_L3) {
237 case ENA_ADMIN_RSS_L3_SA:
238 rss_hf |= RTE_ETH_RSS_L3_SRC_ONLY;
240 case ENA_ADMIN_RSS_L3_DA:
241 rss_hf |= RTE_ETH_RSS_L3_DST_ONLY;
247 /* Check if only DA or SA is being used for L4. */
248 switch (fields & ENA_HF_RSS_ALL_L4) {
249 case ENA_ADMIN_RSS_L4_SP:
250 rss_hf |= RTE_ETH_RSS_L4_SRC_ONLY;
252 case ENA_ADMIN_RSS_L4_DP:
253 rss_hf |= RTE_ETH_RSS_L4_DST_ONLY;
262 static uint16_t ena_eth_hf_to_admin_hf(enum ena_admin_flow_hash_proto proto,
265 uint16_t fields_mask = 0;
267 /* L2 always uses source and destination addresses. */
268 fields_mask = ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA;
270 /* Determine which fields of L3 should be used. */
271 switch (rss_hf & (RTE_ETH_RSS_L3_SRC_ONLY | RTE_ETH_RSS_L3_DST_ONLY)) {
272 case RTE_ETH_RSS_L3_DST_ONLY:
273 fields_mask |= ENA_ADMIN_RSS_L3_DA;
275 case RTE_ETH_RSS_L3_SRC_ONLY:
276 fields_mask |= ENA_ADMIN_RSS_L3_SA;
280 * If SRC nor DST aren't set, it means both of them should be
283 fields_mask |= ENA_HF_RSS_ALL_L3;
286 /* Determine which fields of L4 should be used. */
287 switch (rss_hf & (RTE_ETH_RSS_L4_SRC_ONLY | RTE_ETH_RSS_L4_DST_ONLY)) {
288 case RTE_ETH_RSS_L4_DST_ONLY:
289 fields_mask |= ENA_ADMIN_RSS_L4_DP;
291 case RTE_ETH_RSS_L4_SRC_ONLY:
292 fields_mask |= ENA_ADMIN_RSS_L4_SP;
296 * If SRC nor DST aren't set, it means both of them should be
299 fields_mask |= ENA_HF_RSS_ALL_L4;
302 /* Return appropriate hash fields. */
304 case ENA_ADMIN_RSS_TCP4:
305 return ENA_HF_RSS_TCP4 & fields_mask;
306 case ENA_ADMIN_RSS_UDP4:
307 return ENA_HF_RSS_UDP4 & fields_mask;
308 case ENA_ADMIN_RSS_TCP6:
309 return ENA_HF_RSS_TCP6 & fields_mask;
310 case ENA_ADMIN_RSS_UDP6:
311 return ENA_HF_RSS_UDP6 & fields_mask;
312 case ENA_ADMIN_RSS_IP4:
313 return ENA_HF_RSS_IP4 & fields_mask;
314 case ENA_ADMIN_RSS_IP6:
315 return ENA_HF_RSS_IP6 & fields_mask;
316 case ENA_ADMIN_RSS_IP4_FRAG:
317 return ENA_HF_RSS_IP4_FRAG & fields_mask;
318 case ENA_ADMIN_RSS_NOT_IP:
319 return ENA_HF_RSS_NOT_IP & fields_mask;
320 case ENA_ADMIN_RSS_TCP6_EX:
321 return ENA_HF_RSS_TCP6_EX & fields_mask;
322 case ENA_ADMIN_RSS_IP6_EX:
323 return ENA_HF_RSS_IP6_EX & fields_mask;
331 static int ena_set_hash_fields(struct ena_com_dev *ena_dev, uint64_t rss_hf)
333 struct ena_admin_proto_input selected_fields[ENA_ADMIN_RSS_PROTO_NUM] = {};
336 /* Turn on appropriate fields for each requested packet type */
337 if ((rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_TCP) != 0)
338 selected_fields[ENA_ADMIN_RSS_TCP4].fields =
339 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_TCP4, rss_hf);
341 if ((rss_hf & RTE_ETH_RSS_NONFRAG_IPV4_UDP) != 0)
342 selected_fields[ENA_ADMIN_RSS_UDP4].fields =
343 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_UDP4, rss_hf);
345 if ((rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_TCP) != 0)
346 selected_fields[ENA_ADMIN_RSS_TCP6].fields =
347 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_TCP6, rss_hf);
349 if ((rss_hf & RTE_ETH_RSS_NONFRAG_IPV6_UDP) != 0)
350 selected_fields[ENA_ADMIN_RSS_UDP6].fields =
351 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_UDP6, rss_hf);
353 if ((rss_hf & RTE_ETH_RSS_IPV4) != 0)
354 selected_fields[ENA_ADMIN_RSS_IP4].fields =
355 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_IP4, rss_hf);
357 if ((rss_hf & RTE_ETH_RSS_IPV6) != 0)
358 selected_fields[ENA_ADMIN_RSS_IP6].fields =
359 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_IP6, rss_hf);
361 if ((rss_hf & RTE_ETH_RSS_FRAG_IPV4) != 0)
362 selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
363 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_IP4_FRAG, rss_hf);
365 if ((rss_hf & RTE_ETH_RSS_L2_PAYLOAD) != 0)
366 selected_fields[ENA_ADMIN_RSS_NOT_IP].fields =
367 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_NOT_IP, rss_hf);
369 if ((rss_hf & RTE_ETH_RSS_IPV6_TCP_EX) != 0)
370 selected_fields[ENA_ADMIN_RSS_TCP6_EX].fields =
371 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_TCP6_EX, rss_hf);
373 if ((rss_hf & RTE_ETH_RSS_IPV6_EX) != 0)
374 selected_fields[ENA_ADMIN_RSS_IP6_EX].fields =
375 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_IP6_EX, rss_hf);
377 /* Try to write them to the device */
378 for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) {
379 rc = ena_com_fill_hash_ctrl(ena_dev,
380 (enum ena_admin_flow_hash_proto)i,
381 selected_fields[i].fields);
382 if (unlikely(rc != 0)) {
384 "Failed to set ENA HF %d with fields %" PRIu16 "\n",
385 i, selected_fields[i].fields);
393 static int ena_rss_hash_set(struct ena_com_dev *ena_dev,
394 struct rte_eth_rss_conf *rss_conf,
395 bool default_allowed)
397 uint8_t hw_rss_key[ENA_HASH_KEY_SIZE];
401 if (rss_conf->rss_key != NULL) {
402 /* Reorder the RSS key bytes for the hardware requirements. */
403 ena_reorder_rss_hash_key(hw_rss_key, rss_conf->rss_key,
405 rss_key = hw_rss_key;
410 /* If the rss_key is NULL, then the randomized key will be used. */
411 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ,
412 rss_key, ENA_HASH_KEY_SIZE, 0);
413 if (rc != 0 && !(default_allowed && rc == ENA_COM_UNSUPPORTED)) {
415 "Failed to set RSS hash function in the device\n");
419 rc = ena_set_hash_fields(ena_dev, rss_conf->rss_hf);
420 if (rc == ENA_COM_UNSUPPORTED) {
421 if (rss_conf->rss_key == NULL && !default_allowed) {
423 "Setting RSS hash fields is not supported\n");
427 "Setting RSS hash fields is not supported. Using default values: 0x%" PRIx64 "\n",
428 (uint64_t)(ENA_ALL_RSS_HF));
429 } else if (rc != 0) {
430 PMD_DRV_LOG(ERR, "Failed to set RSS hash fields\n");
437 /* ENA HW interprets the RSS key in reverse bytes order. Because of that, the
438 * key must be processed upon interaction with ena_com layer.
440 static void ena_reorder_rss_hash_key(uint8_t *reordered_key,
446 for (i = 0, rev_i = key_size - 1; i < key_size; ++i, --rev_i)
447 reordered_key[i] = key[rev_i];
450 static int ena_get_rss_hash_key(struct ena_com_dev *ena_dev, uint8_t *rss_key)
452 uint8_t hw_rss_key[ENA_HASH_KEY_SIZE];
455 /* The default RSS hash key cannot be retrieved from the HW. Unless it's
456 * explicitly set, this operation shouldn't be supported.
458 if (ena_dev->rss.hash_key == NULL) {
460 "Retrieving default RSS hash key is not supported\n");
464 rc = ena_com_get_hash_key(ena_dev, hw_rss_key);
468 ena_reorder_rss_hash_key(rss_key, hw_rss_key, ENA_HASH_KEY_SIZE);
473 int ena_rss_configure(struct ena_adapter *adapter)
475 struct rte_eth_rss_conf *rss_conf;
476 struct ena_com_dev *ena_dev;
479 ena_dev = &adapter->ena_dev;
480 rss_conf = &adapter->edev_data->dev_conf.rx_adv_conf.rss_conf;
482 if (adapter->edev_data->nb_rx_queues == 0)
485 /* Restart the indirection table. The number of queues could change
486 * between start/stop calls, so it must be reinitialized with default
489 rc = ena_fill_indirect_table_default(ena_dev, ENA_RX_RSS_TABLE_SIZE,
490 adapter->edev_data->nb_rx_queues);
491 if (unlikely(rc != 0)) {
493 "Failed to fill indirection table with default values\n");
497 rc = ena_com_indirect_table_set(ena_dev);
498 if (unlikely(rc != 0 && rc != ENA_COM_UNSUPPORTED)) {
500 "Failed to set indirection table in the device\n");
504 rc = ena_rss_hash_set(ena_dev, rss_conf, true);
505 if (unlikely(rc != 0)) {
506 PMD_DRV_LOG(ERR, "Failed to set RSS hash\n");
510 PMD_DRV_LOG(DEBUG, "RSS configured for port %d\n",
511 adapter->edev_data->port_id);
516 int ena_rss_hash_update(struct rte_eth_dev *dev,
517 struct rte_eth_rss_conf *rss_conf)
519 struct ena_adapter *adapter = dev->data->dev_private;
522 rte_spinlock_lock(&adapter->admin_lock);
523 rc = ena_rss_hash_set(&adapter->ena_dev, rss_conf, false);
524 rte_spinlock_unlock(&adapter->admin_lock);
525 if (unlikely(rc != 0)) {
526 PMD_DRV_LOG(ERR, "Failed to set RSS hash\n");
533 int ena_rss_hash_conf_get(struct rte_eth_dev *dev,
534 struct rte_eth_rss_conf *rss_conf)
536 struct ena_adapter *adapter = dev->data->dev_private;
537 struct ena_com_dev *ena_dev = &adapter->ena_dev;
538 enum ena_admin_flow_hash_proto proto;
542 static bool warn_once;
544 if (!(dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_RSS_HASH)) {
545 PMD_DRV_LOG(ERR, "RSS was not configured for the PMD\n");
549 if (rss_conf->rss_key != NULL) {
550 rc = ena_get_rss_hash_key(ena_dev, rss_conf->rss_key);
551 if (unlikely(rc != 0)) {
553 "Cannot retrieve RSS hash key, err: %d\n",
559 for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; ++i) {
560 proto = (enum ena_admin_flow_hash_proto)i;
561 rte_spinlock_lock(&adapter->admin_lock);
562 rc = ena_com_get_hash_ctrl(ena_dev, proto, &admin_hf);
563 rte_spinlock_unlock(&adapter->admin_lock);
564 if (rc == ENA_COM_UNSUPPORTED) {
565 /* As some devices may support only reading rss hash
566 * key and not the hash ctrl, we want to notify the
567 * caller that this feature is only partially supported
568 * and do not return an error - the caller could be
569 * interested only in the key value.
573 "Reading hash control from the device is not supported. .rss_hf will contain a default value.\n");
576 rss_hf = ENA_ALL_RSS_HF;
578 } else if (rc != 0) {
580 "Failed to retrieve hash ctrl for proto: %d with err: %d\n",
585 rss_hf |= ena_admin_hf_to_eth_hf(proto, admin_hf);
588 rss_conf->rss_hf = rss_hf;