1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2020 Amazon.com, Inc. or its affiliates.
6 #include "ena_ethdev.h"
9 #include <ena_admin_defs.h>
11 #define TEST_BIT(val, bit_shift) ((val) & (1UL << (bit_shift)))
13 #define ENA_HF_RSS_ALL_L2 (ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA)
14 #define ENA_HF_RSS_ALL_L3 (ENA_ADMIN_RSS_L3_SA | ENA_ADMIN_RSS_L3_DA)
15 #define ENA_HF_RSS_ALL_L4 (ENA_ADMIN_RSS_L4_SP | ENA_ADMIN_RSS_L4_DP)
16 #define ENA_HF_RSS_ALL_L3_L4 (ENA_HF_RSS_ALL_L3 | ENA_HF_RSS_ALL_L4)
17 #define ENA_HF_RSS_ALL_L2_L3_L4 (ENA_HF_RSS_ALL_L2 | ENA_HF_RSS_ALL_L3_L4)
19 enum ena_rss_hash_fields {
20 ENA_HF_RSS_TCP4 = ENA_HF_RSS_ALL_L3_L4,
21 ENA_HF_RSS_UDP4 = ENA_HF_RSS_ALL_L3_L4,
22 ENA_HF_RSS_TCP6 = ENA_HF_RSS_ALL_L3_L4,
23 ENA_HF_RSS_UDP6 = ENA_HF_RSS_ALL_L3_L4,
24 ENA_HF_RSS_IP4 = ENA_HF_RSS_ALL_L3,
25 ENA_HF_RSS_IP6 = ENA_HF_RSS_ALL_L3,
26 ENA_HF_RSS_IP4_FRAG = ENA_HF_RSS_ALL_L3,
27 ENA_HF_RSS_NOT_IP = ENA_HF_RSS_ALL_L2,
28 ENA_HF_RSS_TCP6_EX = ENA_HF_RSS_ALL_L3_L4,
29 ENA_HF_RSS_IP6_EX = ENA_HF_RSS_ALL_L3,
32 static int ena_fill_indirect_table_default(struct ena_com_dev *ena_dev,
35 static uint64_t ena_admin_hf_to_eth_hf(enum ena_admin_flow_hash_proto proto,
37 static uint16_t ena_eth_hf_to_admin_hf(enum ena_admin_flow_hash_proto proto,
39 static int ena_set_hash_fields(struct ena_com_dev *ena_dev, uint64_t rss_hf);
40 static int ena_rss_hash_set(struct ena_com_dev *ena_dev,
41 struct rte_eth_rss_conf *rss_conf,
42 bool default_allowed);
43 static void ena_reorder_rss_hash_key(uint8_t *reordered_key,
46 static int ena_get_rss_hash_key(struct ena_com_dev *ena_dev, uint8_t *rss_key);
48 void ena_rss_key_fill(void *key, size_t size)
50 static bool key_generated;
51 static uint8_t default_key[ENA_HASH_KEY_SIZE];
54 RTE_ASSERT(size <= ENA_HASH_KEY_SIZE);
57 for (i = 0; i < ENA_HASH_KEY_SIZE; ++i)
58 default_key[i] = rte_rand() & 0xff;
62 rte_memcpy(key, default_key, size);
65 int ena_rss_reta_update(struct rte_eth_dev *dev,
66 struct rte_eth_rss_reta_entry64 *reta_conf,
69 struct ena_adapter *adapter = dev->data->dev_private;
70 struct ena_com_dev *ena_dev = &adapter->ena_dev;
76 if (reta_size == 0 || reta_conf == NULL)
79 if (!(dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_RSS_HASH)) {
81 "RSS was not configured for the PMD\n");
85 if (reta_size > ENA_RX_RSS_TABLE_SIZE) {
87 "Requested indirection table size (%d) is bigger than supported: %d\n",
88 reta_size, ENA_RX_RSS_TABLE_SIZE);
92 for (i = 0 ; i < reta_size ; i++) {
93 /* Each reta_conf is for 64 entries.
94 * To support 128 we use 2 conf of 64.
96 conf_idx = i / RTE_RETA_GROUP_SIZE;
97 idx = i % RTE_RETA_GROUP_SIZE;
98 if (TEST_BIT(reta_conf[conf_idx].mask, idx)) {
100 ENA_IO_RXQ_IDX(reta_conf[conf_idx].reta[idx]);
102 rc = ena_com_indirect_table_fill_entry(ena_dev, i,
104 if (unlikely(rc != 0)) {
106 "Cannot fill indirection table\n");
112 rte_spinlock_lock(&adapter->admin_lock);
113 rc = ena_com_indirect_table_set(ena_dev);
114 rte_spinlock_unlock(&adapter->admin_lock);
115 if (unlikely(rc != 0)) {
116 PMD_DRV_LOG(ERR, "Cannot set the indirection table\n");
120 PMD_DRV_LOG(DEBUG, "RSS configured %d entries for port %d\n",
121 reta_size, dev->data->port_id);
126 /* Query redirection table. */
127 int ena_rss_reta_query(struct rte_eth_dev *dev,
128 struct rte_eth_rss_reta_entry64 *reta_conf,
131 uint32_t indirect_table[ENA_RX_RSS_TABLE_SIZE] = { 0 };
132 struct ena_adapter *adapter = dev->data->dev_private;
133 struct ena_com_dev *ena_dev = &adapter->ena_dev;
139 if (reta_size == 0 || reta_conf == NULL ||
140 (reta_size > RTE_RETA_GROUP_SIZE && ((reta_conf + 1) == NULL)))
143 if (!(dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_RSS_HASH)) {
145 "RSS was not configured for the PMD\n");
149 rte_spinlock_lock(&adapter->admin_lock);
150 rc = ena_com_indirect_table_get(ena_dev, indirect_table);
151 rte_spinlock_unlock(&adapter->admin_lock);
152 if (unlikely(rc != 0)) {
153 PMD_DRV_LOG(ERR, "Cannot get indirection table\n");
157 for (i = 0 ; i < reta_size ; i++) {
158 reta_conf_idx = i / RTE_RETA_GROUP_SIZE;
159 reta_idx = i % RTE_RETA_GROUP_SIZE;
160 if (TEST_BIT(reta_conf[reta_conf_idx].mask, reta_idx))
161 reta_conf[reta_conf_idx].reta[reta_idx] =
162 ENA_IO_RXQ_IDX_REV(indirect_table[i]);
168 static int ena_fill_indirect_table_default(struct ena_com_dev *ena_dev,
176 for (i = 0; i < tbl_size; ++i) {
178 rc = ena_com_indirect_table_fill_entry(ena_dev, i,
179 ENA_IO_RXQ_IDX(val));
180 if (unlikely(rc != 0)) {
182 "Failed to set %zu indirection table entry with val %" PRIu16 "\n",
191 static uint64_t ena_admin_hf_to_eth_hf(enum ena_admin_flow_hash_proto proto,
196 /* If no fields are activated, then RSS is disabled for this proto */
197 if ((fields & ENA_HF_RSS_ALL_L2_L3_L4) == 0)
200 /* Convert proto to ETH flag */
202 case ENA_ADMIN_RSS_TCP4:
203 rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
205 case ENA_ADMIN_RSS_UDP4:
206 rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
208 case ENA_ADMIN_RSS_TCP6:
209 rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
211 case ENA_ADMIN_RSS_UDP6:
212 rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
214 case ENA_ADMIN_RSS_IP4:
215 rss_hf |= ETH_RSS_IPV4;
217 case ENA_ADMIN_RSS_IP6:
218 rss_hf |= ETH_RSS_IPV6;
220 case ENA_ADMIN_RSS_IP4_FRAG:
221 rss_hf |= ETH_RSS_FRAG_IPV4;
223 case ENA_ADMIN_RSS_NOT_IP:
224 rss_hf |= ETH_RSS_L2_PAYLOAD;
226 case ENA_ADMIN_RSS_TCP6_EX:
227 rss_hf |= ETH_RSS_IPV6_TCP_EX;
229 case ENA_ADMIN_RSS_IP6_EX:
230 rss_hf |= ETH_RSS_IPV6_EX;
236 /* Check if only DA or SA is being used for L3. */
237 switch (fields & ENA_HF_RSS_ALL_L3) {
238 case ENA_ADMIN_RSS_L3_SA:
239 rss_hf |= ETH_RSS_L3_SRC_ONLY;
241 case ENA_ADMIN_RSS_L3_DA:
242 rss_hf |= ETH_RSS_L3_DST_ONLY;
248 /* Check if only DA or SA is being used for L4. */
249 switch (fields & ENA_HF_RSS_ALL_L4) {
250 case ENA_ADMIN_RSS_L4_SP:
251 rss_hf |= ETH_RSS_L4_SRC_ONLY;
253 case ENA_ADMIN_RSS_L4_DP:
254 rss_hf |= ETH_RSS_L4_DST_ONLY;
263 static uint16_t ena_eth_hf_to_admin_hf(enum ena_admin_flow_hash_proto proto,
266 uint16_t fields_mask = 0;
268 /* L2 always uses source and destination addresses. */
269 fields_mask = ENA_ADMIN_RSS_L2_DA | ENA_ADMIN_RSS_L2_SA;
271 /* Determine which fields of L3 should be used. */
272 switch (rss_hf & (ETH_RSS_L3_SRC_ONLY | ETH_RSS_L3_DST_ONLY)) {
273 case ETH_RSS_L3_DST_ONLY:
274 fields_mask |= ENA_ADMIN_RSS_L3_DA;
276 case ETH_RSS_L3_SRC_ONLY:
277 fields_mask |= ENA_ADMIN_RSS_L3_SA;
281 * If SRC nor DST aren't set, it means both of them should be
284 fields_mask |= ENA_HF_RSS_ALL_L3;
287 /* Determine which fields of L4 should be used. */
288 switch (rss_hf & (ETH_RSS_L4_SRC_ONLY | ETH_RSS_L4_DST_ONLY)) {
289 case ETH_RSS_L4_DST_ONLY:
290 fields_mask |= ENA_ADMIN_RSS_L4_DP;
292 case ETH_RSS_L4_SRC_ONLY:
293 fields_mask |= ENA_ADMIN_RSS_L4_SP;
297 * If SRC nor DST aren't set, it means both of them should be
300 fields_mask |= ENA_HF_RSS_ALL_L4;
303 /* Return appropriate hash fields. */
305 case ENA_ADMIN_RSS_TCP4:
306 return ENA_HF_RSS_TCP4 & fields_mask;
307 case ENA_ADMIN_RSS_UDP4:
308 return ENA_HF_RSS_UDP4 & fields_mask;
309 case ENA_ADMIN_RSS_TCP6:
310 return ENA_HF_RSS_TCP6 & fields_mask;
311 case ENA_ADMIN_RSS_UDP6:
312 return ENA_HF_RSS_UDP6 & fields_mask;
313 case ENA_ADMIN_RSS_IP4:
314 return ENA_HF_RSS_IP4 & fields_mask;
315 case ENA_ADMIN_RSS_IP6:
316 return ENA_HF_RSS_IP6 & fields_mask;
317 case ENA_ADMIN_RSS_IP4_FRAG:
318 return ENA_HF_RSS_IP4_FRAG & fields_mask;
319 case ENA_ADMIN_RSS_NOT_IP:
320 return ENA_HF_RSS_NOT_IP & fields_mask;
321 case ENA_ADMIN_RSS_TCP6_EX:
322 return ENA_HF_RSS_TCP6_EX & fields_mask;
323 case ENA_ADMIN_RSS_IP6_EX:
324 return ENA_HF_RSS_IP6_EX & fields_mask;
332 static int ena_set_hash_fields(struct ena_com_dev *ena_dev, uint64_t rss_hf)
334 struct ena_admin_proto_input selected_fields[ENA_ADMIN_RSS_PROTO_NUM] = {};
337 /* Turn on appropriate fields for each requested packet type */
338 if ((rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) != 0)
339 selected_fields[ENA_ADMIN_RSS_TCP4].fields =
340 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_TCP4, rss_hf);
342 if ((rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) != 0)
343 selected_fields[ENA_ADMIN_RSS_UDP4].fields =
344 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_UDP4, rss_hf);
346 if ((rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) != 0)
347 selected_fields[ENA_ADMIN_RSS_TCP6].fields =
348 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_TCP6, rss_hf);
350 if ((rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) != 0)
351 selected_fields[ENA_ADMIN_RSS_UDP6].fields =
352 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_UDP6, rss_hf);
354 if ((rss_hf & ETH_RSS_IPV4) != 0)
355 selected_fields[ENA_ADMIN_RSS_IP4].fields =
356 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_IP4, rss_hf);
358 if ((rss_hf & ETH_RSS_IPV6) != 0)
359 selected_fields[ENA_ADMIN_RSS_IP6].fields =
360 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_IP6, rss_hf);
362 if ((rss_hf & ETH_RSS_FRAG_IPV4) != 0)
363 selected_fields[ENA_ADMIN_RSS_IP4_FRAG].fields =
364 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_IP4_FRAG, rss_hf);
366 if ((rss_hf & ETH_RSS_L2_PAYLOAD) != 0)
367 selected_fields[ENA_ADMIN_RSS_NOT_IP].fields =
368 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_NOT_IP, rss_hf);
370 if ((rss_hf & ETH_RSS_IPV6_TCP_EX) != 0)
371 selected_fields[ENA_ADMIN_RSS_TCP6_EX].fields =
372 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_TCP6_EX, rss_hf);
374 if ((rss_hf & ETH_RSS_IPV6_EX) != 0)
375 selected_fields[ENA_ADMIN_RSS_IP6_EX].fields =
376 ena_eth_hf_to_admin_hf(ENA_ADMIN_RSS_IP6_EX, rss_hf);
378 /* Try to write them to the device */
379 for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; i++) {
380 rc = ena_com_fill_hash_ctrl(ena_dev,
381 (enum ena_admin_flow_hash_proto)i,
382 selected_fields[i].fields);
383 if (unlikely(rc != 0)) {
385 "Failed to set ENA HF %d with fields %" PRIu16 "\n",
386 i, selected_fields[i].fields);
394 static int ena_rss_hash_set(struct ena_com_dev *ena_dev,
395 struct rte_eth_rss_conf *rss_conf,
396 bool default_allowed)
398 uint8_t hw_rss_key[ENA_HASH_KEY_SIZE];
402 if (rss_conf->rss_key != NULL) {
403 /* Reorder the RSS key bytes for the hardware requirements. */
404 ena_reorder_rss_hash_key(hw_rss_key, rss_conf->rss_key,
406 rss_key = hw_rss_key;
411 /* If the rss_key is NULL, then the randomized key will be used. */
412 rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ,
413 rss_key, ENA_HASH_KEY_SIZE, 0);
414 if (rc != 0 && !(default_allowed && rc == ENA_COM_UNSUPPORTED)) {
416 "Failed to set RSS hash function in the device\n");
420 rc = ena_set_hash_fields(ena_dev, rss_conf->rss_hf);
421 if (rc == ENA_COM_UNSUPPORTED) {
422 if (rss_conf->rss_key == NULL && !default_allowed) {
424 "Setting RSS hash fields is not supported\n");
428 "Setting RSS hash fields is not supported. Using default values: 0x%" PRIx64 "\n",
429 (uint64_t)(ENA_ALL_RSS_HF));
430 } else if (rc != 0) {
431 PMD_DRV_LOG(ERR, "Failed to set RSS hash fields\n");
438 /* ENA HW interprets the RSS key in reverse bytes order. Because of that, the
439 * key must be processed upon interaction with ena_com layer.
441 static void ena_reorder_rss_hash_key(uint8_t *reordered_key,
447 for (i = 0, rev_i = key_size - 1; i < key_size; ++i, --rev_i)
448 reordered_key[i] = key[rev_i];
451 static int ena_get_rss_hash_key(struct ena_com_dev *ena_dev, uint8_t *rss_key)
453 uint8_t hw_rss_key[ENA_HASH_KEY_SIZE];
456 /* The default RSS hash key cannot be retrieved from the HW. Unless it's
457 * explicitly set, this operation shouldn't be supported.
459 if (ena_dev->rss.hash_key == NULL) {
461 "Retrieving default RSS hash key is not supported\n");
465 rc = ena_com_get_hash_key(ena_dev, hw_rss_key);
469 ena_reorder_rss_hash_key(rss_key, hw_rss_key, ENA_HASH_KEY_SIZE);
474 int ena_rss_configure(struct ena_adapter *adapter)
476 struct rte_eth_rss_conf *rss_conf;
477 struct ena_com_dev *ena_dev;
480 ena_dev = &adapter->ena_dev;
481 rss_conf = &adapter->edev_data->dev_conf.rx_adv_conf.rss_conf;
483 if (adapter->edev_data->nb_rx_queues == 0)
486 /* Restart the indirection table. The number of queues could change
487 * between start/stop calls, so it must be reinitialized with default
490 rc = ena_fill_indirect_table_default(ena_dev, ENA_RX_RSS_TABLE_SIZE,
491 adapter->edev_data->nb_rx_queues);
492 if (unlikely(rc != 0)) {
494 "Failed to fill indirection table with default values\n");
498 rc = ena_com_indirect_table_set(ena_dev);
499 if (unlikely(rc != 0 && rc != ENA_COM_UNSUPPORTED)) {
501 "Failed to set indirection table in the device\n");
505 rc = ena_rss_hash_set(ena_dev, rss_conf, true);
506 if (unlikely(rc != 0)) {
507 PMD_DRV_LOG(ERR, "Failed to set RSS hash\n");
511 PMD_DRV_LOG(DEBUG, "RSS configured for port %d\n",
512 adapter->edev_data->port_id);
517 int ena_rss_hash_update(struct rte_eth_dev *dev,
518 struct rte_eth_rss_conf *rss_conf)
520 struct ena_adapter *adapter = dev->data->dev_private;
523 rte_spinlock_lock(&adapter->admin_lock);
524 rc = ena_rss_hash_set(&adapter->ena_dev, rss_conf, false);
525 rte_spinlock_unlock(&adapter->admin_lock);
526 if (unlikely(rc != 0)) {
527 PMD_DRV_LOG(ERR, "Failed to set RSS hash\n");
534 int ena_rss_hash_conf_get(struct rte_eth_dev *dev,
535 struct rte_eth_rss_conf *rss_conf)
537 struct ena_adapter *adapter = dev->data->dev_private;
538 struct ena_com_dev *ena_dev = &adapter->ena_dev;
539 enum ena_admin_flow_hash_proto proto;
543 static bool warn_once;
545 if (!(dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_RSS_HASH)) {
546 PMD_DRV_LOG(ERR, "RSS was not configured for the PMD\n");
550 if (rss_conf->rss_key != NULL) {
551 rc = ena_get_rss_hash_key(ena_dev, rss_conf->rss_key);
552 if (unlikely(rc != 0)) {
554 "Cannot retrieve RSS hash key, err: %d\n",
560 for (i = 0; i < ENA_ADMIN_RSS_PROTO_NUM; ++i) {
561 proto = (enum ena_admin_flow_hash_proto)i;
562 rte_spinlock_lock(&adapter->admin_lock);
563 rc = ena_com_get_hash_ctrl(ena_dev, proto, &admin_hf);
564 rte_spinlock_unlock(&adapter->admin_lock);
565 if (rc == ENA_COM_UNSUPPORTED) {
566 /* As some devices may support only reading rss hash
567 * key and not the hash ctrl, we want to notify the
568 * caller that this feature is only partially supported
569 * and do not return an error - the caller could be
570 * interested only in the key value.
574 "Reading hash control from the device is not supported. .rss_hf will contain a default value.\n");
577 rss_hf = ENA_ALL_RSS_HF;
579 } else if (rc != 0) {
581 "Failed to retrieve hash ctrl for proto: %d with err: %d\n",
586 rss_hf |= ena_admin_hf_to_eth_hf(proto, admin_hf);
589 rss_conf->rss_hf = rss_hf;