From 5935cd8c47d3787c534c7b81893944f2c7866876 Mon Sep 17 00:00:00 2001 From: Andrew Rybchenko Date: Tue, 29 Nov 2016 16:18:53 +0000 Subject: [PATCH] net/sfc/base: import RSS support EFSYS_OPT_RX_SCALE should be enabled to use it. From Solarflare Communications Inc. Signed-off-by: Andrew Rybchenko Reviewed-by: Ferruh Yigit --- drivers/net/sfc/base/ef10_filter.c | 10 + drivers/net/sfc/base/ef10_impl.h | 29 ++ drivers/net/sfc/base/ef10_rx.c | 443 ++++++++++++++++++++++++++++ drivers/net/sfc/base/efx.h | 67 +++++ drivers/net/sfc/base/efx_check.h | 7 + drivers/net/sfc/base/efx_ev.c | 11 + drivers/net/sfc/base/efx_filter.c | 4 + drivers/net/sfc/base/efx_impl.h | 14 + drivers/net/sfc/base/efx_rx.c | 456 +++++++++++++++++++++++++++++ 9 files changed, 1041 insertions(+) diff --git a/drivers/net/sfc/base/ef10_filter.c b/drivers/net/sfc/base/ef10_filter.c index 451416046d..c161977a5d 100644 --- a/drivers/net/sfc/base/ef10_filter.c +++ b/drivers/net/sfc/base/ef10_filter.c @@ -562,6 +562,10 @@ ef10_filter_add_internal( EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || enp->en_family == EFX_FAMILY_MEDFORD); +#if EFSYS_OPT_RX_SCALE + spec->efs_rss_context = enp->en_rss_context; +#endif + hash = ef10_filter_hash(spec); /* @@ -1448,8 +1452,14 @@ ef10_filter_default_rxq_set( { ef10_filter_table_t *table = enp->en_filter.ef_ef10_filter_table; +#if EFSYS_OPT_RX_SCALE + EFSYS_ASSERT((using_rss == B_FALSE) || + (enp->en_rss_context != EF10_RSS_CONTEXT_INVALID)); + table->eft_using_rss = using_rss; +#else EFSYS_ASSERT(using_rss == B_FALSE); table->eft_using_rss = B_FALSE; +#endif table->eft_default_rxq = erp; } diff --git a/drivers/net/sfc/base/ef10_impl.h b/drivers/net/sfc/base/ef10_impl.h index 9d246514a4..0e5561e89b 100644 --- a/drivers/net/sfc/base/ef10_impl.h +++ b/drivers/net/sfc/base/ef10_impl.h @@ -581,6 +581,35 @@ ef10_rx_scatter_enable( #endif /* EFSYS_OPT_RX_SCATTER */ +#if EFSYS_OPT_RX_SCALE + +extern __checkReturn efx_rc_t +ef10_rx_scale_mode_set( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t alg, + __in efx_rx_hash_type_t type, + __in boolean_t insert); + +extern __checkReturn efx_rc_t +ef10_rx_scale_key_set( + __in efx_nic_t *enp, + __in_ecount(n) uint8_t *key, + __in size_t n); + +extern __checkReturn efx_rc_t +ef10_rx_scale_tbl_set( + __in efx_nic_t *enp, + __in_ecount(n) unsigned int *table, + __in size_t n); + +extern __checkReturn uint32_t +ef10_rx_prefix_hash( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t func, + __in uint8_t *buffer); + +#endif /* EFSYS_OPT_RX_SCALE */ + extern __checkReturn efx_rc_t ef10_rx_prefix_pktlen( __in efx_nic_t *enp, diff --git a/drivers/net/sfc/base/ef10_rx.c b/drivers/net/sfc/base/ef10_rx.c index 95a182b9b9..09a6314821 100644 --- a/drivers/net/sfc/base/ef10_rx.c +++ b/drivers/net/sfc/base/ef10_rx.c @@ -150,11 +150,325 @@ fail1: return (rc); } +#if EFSYS_OPT_RX_SCALE +static __checkReturn efx_rc_t +efx_mcdi_rss_context_alloc( + __in efx_nic_t *enp, + __in efx_rx_scale_support_t scale_support, + __in uint32_t num_queues, + __out uint32_t *rss_contextp) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, + MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN)]; + uint32_t rss_context; + uint32_t context_type; + efx_rc_t rc; + + if (num_queues > EFX_MAXRSS) { + rc = EINVAL; + goto fail1; + } + + switch (scale_support) { + case EFX_RX_SCALE_EXCLUSIVE: + context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; + break; + case EFX_RX_SCALE_SHARED: + context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; + break; + default: + rc = EINVAL; + goto fail2; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; + + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, + EVB_PORT_ID_ASSIGNED); + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); + /* NUM_QUEUES is only used to validate indirection table offsets */ + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail3; + } + + if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { + rc = EMSGSIZE; + goto fail4; + } + + rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); + if (rss_context == EF10_RSS_CONTEXT_INVALID) { + rc = ENOENT; + goto fail5; + } + + *rss_contextp = rss_context; + + return (0); + +fail5: + EFSYS_PROBE(fail5); +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE +static efx_rc_t +efx_mcdi_rss_context_free( + __in efx_nic_t *enp, + __in uint32_t rss_context) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_FREE_IN_LEN, + MC_CMD_RSS_CONTEXT_FREE_OUT_LEN)]; + efx_rc_t rc; + + if (rss_context == EF10_RSS_CONTEXT_INVALID) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; + + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); + + efx_mcdi_execute_quiet(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE +static efx_rc_t +efx_mcdi_rss_context_set_flags( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in efx_rx_hash_type_t type) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, + MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN)]; + efx_rc_t rc; + + if (rss_context == EF10_RSS_CONTEXT_INVALID) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; + + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, + rss_context); + + MCDI_IN_POPULATE_DWORD_4(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, + RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, + (type & (1U << EFX_RX_HASH_IPV4)) ? 1 : 0, + RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, + (type & (1U << EFX_RX_HASH_TCPIPV4)) ? 1 : 0, + RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, + (type & (1U << EFX_RX_HASH_IPV6)) ? 1 : 0, + RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, + (type & (1U << EFX_RX_HASH_TCPIPV6)) ? 1 : 0); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE +static efx_rc_t +efx_mcdi_rss_context_set_key( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) uint8_t *key, + __in size_t n) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, + MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN)]; + efx_rc_t rc; + + if (rss_context == EF10_RSS_CONTEXT_INVALID) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; + + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, + rss_context); + + EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); + if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { + rc = EINVAL; + goto fail2; + } + + memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), + key, n); + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail3; + } + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE +static efx_rc_t +efx_mcdi_rss_context_set_table( + __in efx_nic_t *enp, + __in uint32_t rss_context, + __in_ecount(n) unsigned int *table, + __in size_t n) +{ + efx_mcdi_req_t req; + uint8_t payload[MAX(MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, + MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN)]; + uint8_t *req_table; + int i, rc; + + if (rss_context == EF10_RSS_CONTEXT_INVALID) { + rc = EINVAL; + goto fail1; + } + + (void) memset(payload, 0, sizeof (payload)); + req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; + req.emr_in_buf = payload; + req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; + req.emr_out_buf = payload; + req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; + + MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, + rss_context); + + req_table = + MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); + + for (i = 0; + i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; + i++) { + req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; + } + + efx_mcdi_execute(enp, &req); + + if (req.emr_rc != 0) { + rc = req.emr_rc; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + __checkReturn efx_rc_t ef10_rx_init( __in efx_nic_t *enp) { +#if EFSYS_OPT_RX_SCALE + + if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, + &enp->en_rss_context) == 0) { + /* + * Allocated an exclusive RSS context, which allows both the + * indirection table and key to be modified. + */ + enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE; + enp->en_hash_support = EFX_RX_HASH_AVAILABLE; + } else { + /* + * Failed to allocate an exclusive RSS context. Continue + * operation without support for RSS. The pseudo-header in + * received packets will not contain a Toeplitz hash value. + */ + enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE; + enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; + } + +#endif /* EFSYS_OPT_RX_SCALE */ return (0); } @@ -170,6 +484,104 @@ ef10_rx_scatter_enable( } #endif /* EFSYS_OPT_RX_SCATTER */ +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +ef10_rx_scale_mode_set( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t alg, + __in efx_rx_hash_type_t type, + __in boolean_t insert) +{ + efx_rc_t rc; + + EFSYS_ASSERT3U(alg, ==, EFX_RX_HASHALG_TOEPLITZ); + EFSYS_ASSERT3U(insert, ==, B_TRUE); + + if ((alg != EFX_RX_HASHALG_TOEPLITZ) || (insert == B_FALSE)) { + rc = EINVAL; + goto fail1; + } + + if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { + rc = ENOTSUP; + goto fail2; + } + + if ((rc = efx_mcdi_rss_context_set_flags(enp, + enp->en_rss_context, type)) != 0) + goto fail3; + + return (0); + +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +ef10_rx_scale_key_set( + __in efx_nic_t *enp, + __in_ecount(n) uint8_t *key, + __in size_t n) +{ + efx_rc_t rc; + + if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { + rc = ENOTSUP; + goto fail1; + } + + if ((rc = efx_mcdi_rss_context_set_key(enp, + enp->en_rss_context, key, n)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +ef10_rx_scale_tbl_set( + __in efx_nic_t *enp, + __in_ecount(n) unsigned int *table, + __in size_t n) +{ + efx_rc_t rc; + + if (enp->en_rss_support == EFX_RX_SCALE_UNAVAILABLE) { + rc = ENOTSUP; + goto fail1; + } + + if ((rc = efx_mcdi_rss_context_set_table(enp, + enp->en_rss_context, table, n)) != 0) + goto fail2; + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + /* * EF10 RX pseudo-header @@ -210,6 +622,29 @@ ef10_rx_prefix_pktlen( return (0); } +#if EFSYS_OPT_RX_SCALE + __checkReturn uint32_t +ef10_rx_prefix_hash( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t func, + __in uint8_t *buffer) +{ + _NOTE(ARGUNUSED(enp)) + + switch (func) { + case EFX_RX_HASHALG_TOEPLITZ: + return (buffer[0] | + (buffer[1] << 8) | + (buffer[2] << 16) | + (buffer[3] << 24)); + + default: + EFSYS_ASSERT(0); + return (0); + } +} +#endif /* EFSYS_OPT_RX_SCALE */ + void ef10_rx_qpost( __in efx_rxq_t *erp, @@ -402,7 +837,15 @@ ef10_rx_qdestroy( ef10_rx_fini( __in efx_nic_t *enp) { +#if EFSYS_OPT_RX_SCALE + if (enp->en_rss_support != EFX_RX_SCALE_UNAVAILABLE) { + (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); + } + enp->en_rss_context = 0; + enp->en_rss_support = EFX_RX_SCALE_UNAVAILABLE; +#else _NOTE(ARGUNUSED(enp)) +#endif /* EFSYS_OPT_RX_SCALE */ } #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ diff --git a/drivers/net/sfc/base/efx.h b/drivers/net/sfc/base/efx.h index 1013d5df17..f140a89f6e 100644 --- a/drivers/net/sfc/base/efx.h +++ b/drivers/net/sfc/base/efx.h @@ -1388,6 +1388,73 @@ efx_rx_scatter_enable( __in unsigned int buf_size); #endif /* EFSYS_OPT_RX_SCATTER */ +#if EFSYS_OPT_RX_SCALE + +typedef enum efx_rx_hash_alg_e { + EFX_RX_HASHALG_LFSR = 0, + EFX_RX_HASHALG_TOEPLITZ +} efx_rx_hash_alg_t; + +typedef enum efx_rx_hash_type_e { + EFX_RX_HASH_IPV4 = 0, + EFX_RX_HASH_TCPIPV4, + EFX_RX_HASH_IPV6, + EFX_RX_HASH_TCPIPV6, +} efx_rx_hash_type_t; + +typedef enum efx_rx_hash_support_e { + EFX_RX_HASH_UNAVAILABLE = 0, /* Hardware hash not inserted */ + EFX_RX_HASH_AVAILABLE /* Insert hash with/without RSS */ +} efx_rx_hash_support_t; + +#define EFX_RSS_TBL_SIZE 128 /* Rows in RX indirection table */ +#define EFX_MAXRSS 64 /* RX indirection entry range */ +#define EFX_MAXRSS_LEGACY 16 /* See bug16611 and bug17213 */ + +typedef enum efx_rx_scale_support_e { + EFX_RX_SCALE_UNAVAILABLE = 0, /* Not supported */ + EFX_RX_SCALE_EXCLUSIVE, /* Writable key/indirection table */ + EFX_RX_SCALE_SHARED /* Read-only key/indirection table */ +} efx_rx_scale_support_t; + +extern __checkReturn efx_rc_t +efx_rx_hash_support_get( + __in efx_nic_t *enp, + __out efx_rx_hash_support_t *supportp); + + +extern __checkReturn efx_rc_t +efx_rx_scale_support_get( + __in efx_nic_t *enp, + __out efx_rx_scale_support_t *supportp); + +extern __checkReturn efx_rc_t +efx_rx_scale_mode_set( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t alg, + __in efx_rx_hash_type_t type, + __in boolean_t insert); + +extern __checkReturn efx_rc_t +efx_rx_scale_tbl_set( + __in efx_nic_t *enp, + __in_ecount(n) unsigned int *table, + __in size_t n); + +extern __checkReturn efx_rc_t +efx_rx_scale_key_set( + __in efx_nic_t *enp, + __in_ecount(n) uint8_t *key, + __in size_t n); + +extern __checkReturn uint32_t +efx_pseudo_hdr_hash_get( + __in efx_rxq_t *erp, + __in efx_rx_hash_alg_t func, + __in uint8_t *buffer); + +#endif /* EFSYS_OPT_RX_SCALE */ + extern __checkReturn efx_rc_t efx_pseudo_hdr_pkt_length_get( __in efx_rxq_t *erp, diff --git a/drivers/net/sfc/base/efx_check.h b/drivers/net/sfc/base/efx_check.h index 91a764fcaf..ac248b3bed 100644 --- a/drivers/net/sfc/base/efx_check.h +++ b/drivers/net/sfc/base/efx_check.h @@ -244,6 +244,13 @@ # error "RX_HDR_SPLIT is obsolete and is not supported" #endif +#if EFSYS_OPT_RX_SCALE +/* Support receive scaling (RSS) */ +# if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) +# error "RX_SCALE requires SIENA or HUNTINGTON or MEDFORD" +# endif +#endif /* EFSYS_OPT_RX_SCALE */ + #if EFSYS_OPT_RX_SCATTER /* Support receive scatter DMA */ # if !(EFSYS_OPT_SIENA || EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD) diff --git a/drivers/net/sfc/base/efx_ev.c b/drivers/net/sfc/base/efx_ev.c index a6671247a5..9f6a309111 100644 --- a/drivers/net/sfc/base/efx_ev.c +++ b/drivers/net/sfc/base/efx_ev.c @@ -1285,6 +1285,13 @@ siena_ev_qcreate( rc = EINVAL; goto fail2; } +#if EFSYS_OPT_RX_SCALE + if (enp->en_intr.ei_type == EFX_INTR_LINE && + index >= EFX_MAXRSS_LEGACY) { + rc = EINVAL; + goto fail3; + } +#endif for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS); size++) if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS)) @@ -1325,6 +1332,10 @@ siena_ev_qcreate( fail4: EFSYS_PROBE(fail4); +#if EFSYS_OPT_RX_SCALE +fail3: + EFSYS_PROBE(fail3); +#endif fail2: EFSYS_PROBE(fail2); fail1: diff --git a/drivers/net/sfc/base/efx_filter.c b/drivers/net/sfc/base/efx_filter.c index c878b784bc..a20efaf96b 100644 --- a/drivers/net/sfc/base/efx_filter.c +++ b/drivers/net/sfc/base/efx_filter.c @@ -116,6 +116,10 @@ efx_filter_remove( EFSYS_ASSERT3P(spec, !=, NULL); EFSYS_ASSERT3U(spec->efs_flags, &, EFX_FILTER_FLAG_RX); +#if EFSYS_OPT_RX_SCALE + spec->efs_rss_context = enp->en_rss_context; +#endif + return (efop->efo_delete(enp, spec)); } diff --git a/drivers/net/sfc/base/efx_impl.h b/drivers/net/sfc/base/efx_impl.h index b7eeee7d25..e88e8a9d1a 100644 --- a/drivers/net/sfc/base/efx_impl.h +++ b/drivers/net/sfc/base/efx_impl.h @@ -151,6 +151,15 @@ typedef struct efx_rx_ops_s { #if EFSYS_OPT_RX_SCATTER efx_rc_t (*erxo_scatter_enable)(efx_nic_t *, unsigned int); #endif +#if EFSYS_OPT_RX_SCALE + efx_rc_t (*erxo_scale_mode_set)(efx_nic_t *, efx_rx_hash_alg_t, + efx_rx_hash_type_t, boolean_t); + efx_rc_t (*erxo_scale_key_set)(efx_nic_t *, uint8_t *, size_t); + efx_rc_t (*erxo_scale_tbl_set)(efx_nic_t *, unsigned int *, + size_t); + uint32_t (*erxo_prefix_hash)(efx_nic_t *, efx_rx_hash_alg_t, + uint8_t *); +#endif /* EFSYS_OPT_RX_SCALE */ efx_rc_t (*erxo_prefix_pktlen)(efx_nic_t *, uint8_t *, uint16_t *); void (*erxo_qpost)(efx_rxq_t *, efsys_dma_addr_t *, size_t, @@ -461,6 +470,11 @@ struct efx_nic_s { #if EFSYS_OPT_MCDI efx_mcdi_t en_mcdi; #endif /* EFSYS_OPT_MCDI */ +#if EFSYS_OPT_RX_SCALE + efx_rx_hash_support_t en_hash_support; + efx_rx_scale_support_t en_rss_support; + uint32_t en_rss_context; +#endif /* EFSYS_OPT_RX_SCALE */ uint32_t en_vport_id; union { #if EFSYS_OPT_SIENA diff --git a/drivers/net/sfc/base/efx_rx.c b/drivers/net/sfc/base/efx_rx.c index 7cf07e9f98..9a01b75e7d 100644 --- a/drivers/net/sfc/base/efx_rx.c +++ b/drivers/net/sfc/base/efx_rx.c @@ -49,6 +49,34 @@ siena_rx_scatter_enable( __in unsigned int buf_size); #endif /* EFSYS_OPT_RX_SCATTER */ +#if EFSYS_OPT_RX_SCALE +static __checkReturn efx_rc_t +siena_rx_scale_mode_set( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t alg, + __in efx_rx_hash_type_t type, + __in boolean_t insert); + +static __checkReturn efx_rc_t +siena_rx_scale_key_set( + __in efx_nic_t *enp, + __in_ecount(n) uint8_t *key, + __in size_t n); + +static __checkReturn efx_rc_t +siena_rx_scale_tbl_set( + __in efx_nic_t *enp, + __in_ecount(n) unsigned int *table, + __in size_t n); + +static __checkReturn uint32_t +siena_rx_prefix_hash( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t func, + __in uint8_t *buffer); + +#endif /* EFSYS_OPT_RX_SCALE */ + static __checkReturn efx_rc_t siena_rx_prefix_pktlen( __in efx_nic_t *enp, @@ -103,6 +131,12 @@ static const efx_rx_ops_t __efx_rx_siena_ops = { siena_rx_fini, /* erxo_fini */ #if EFSYS_OPT_RX_SCATTER siena_rx_scatter_enable, /* erxo_scatter_enable */ +#endif +#if EFSYS_OPT_RX_SCALE + siena_rx_scale_mode_set, /* erxo_scale_mode_set */ + siena_rx_scale_key_set, /* erxo_scale_key_set */ + siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ + siena_rx_prefix_hash, /* erxo_prefix_hash */ #endif siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ siena_rx_qpost, /* erxo_qpost */ @@ -120,6 +154,12 @@ static const efx_rx_ops_t __efx_rx_ef10_ops = { ef10_rx_fini, /* erxo_fini */ #if EFSYS_OPT_RX_SCATTER ef10_rx_scatter_enable, /* erxo_scatter_enable */ +#endif +#if EFSYS_OPT_RX_SCALE + ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ + ef10_rx_scale_key_set, /* erxo_scale_key_set */ + ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ + ef10_rx_prefix_hash, /* erxo_prefix_hash */ #endif ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ ef10_rx_qpost, /* erxo_qpost */ @@ -238,6 +278,136 @@ fail1: } #endif /* EFSYS_OPT_RX_SCATTER */ +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +efx_rx_hash_support_get( + __in efx_nic_t *enp, + __out efx_rx_hash_support_t *supportp) +{ + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if (supportp == NULL) { + rc = EINVAL; + goto fail1; + } + + /* Report if resources are available to insert RX hash value */ + *supportp = enp->en_hash_support; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_rx_scale_support_get( + __in efx_nic_t *enp, + __out efx_rx_scale_support_t *supportp) +{ + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if (supportp == NULL) { + rc = EINVAL; + goto fail1; + } + + /* Report if resources are available to support RSS */ + *supportp = enp->en_rss_support; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} + + __checkReturn efx_rc_t +efx_rx_scale_mode_set( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t alg, + __in efx_rx_hash_type_t type, + __in boolean_t insert) +{ + const efx_rx_ops_t *erxop = enp->en_erxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if (erxop->erxo_scale_mode_set != NULL) { + if ((rc = erxop->erxo_scale_mode_set(enp, alg, + type, insert)) != 0) + goto fail1; + } + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +efx_rx_scale_key_set( + __in efx_nic_t *enp, + __in_ecount(n) uint8_t *key, + __in size_t n) +{ + const efx_rx_ops_t *erxop = enp->en_erxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if ((rc = erxop->erxo_scale_key_set(enp, key, n)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + +#if EFSYS_OPT_RX_SCALE + __checkReturn efx_rc_t +efx_rx_scale_tbl_set( + __in efx_nic_t *enp, + __in_ecount(n) unsigned int *table, + __in size_t n) +{ + const efx_rx_ops_t *erxop = enp->en_erxop; + efx_rc_t rc; + + EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); + EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); + + if ((rc = erxop->erxo_scale_tbl_set(enp, table, n)) != 0) + goto fail1; + + return (0); + +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif /* EFSYS_OPT_RX_SCALE */ + void efx_rx_qpost( __in efx_rxq_t *erp, @@ -380,6 +550,23 @@ efx_pseudo_hdr_pkt_length_get( return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); } +#if EFSYS_OPT_RX_SCALE + __checkReturn uint32_t +efx_pseudo_hdr_hash_get( + __in efx_rxq_t *erp, + __in efx_rx_hash_alg_t func, + __in uint8_t *buffer) +{ + efx_nic_t *enp = erp->er_enp; + const efx_rx_ops_t *erxop = enp->en_erxop; + + EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); + + EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); + return (erxop->erxo_prefix_hash(enp, func, buffer)); +} +#endif /* EFSYS_OPT_RX_SCALE */ + #if EFSYS_OPT_SIENA static __checkReturn efx_rc_t @@ -407,6 +594,14 @@ siena_rx_init( index, &oword, B_TRUE); } +#if EFSYS_OPT_RX_SCALE + /* The RSS key and indirection table are writable. */ + enp->en_rss_support = EFX_RX_SCALE_EXCLUSIVE; + + /* Hardware can insert RX hash with/without RSS */ + enp->en_hash_support = EFX_RX_HASH_AVAILABLE; +#endif /* EFSYS_OPT_RX_SCALE */ + return (0); } @@ -515,6 +710,241 @@ fail1: } while (B_FALSE) +#if EFSYS_OPT_RX_SCALE + +static __checkReturn efx_rc_t +siena_rx_scale_mode_set( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t alg, + __in efx_rx_hash_type_t type, + __in boolean_t insert) +{ + efx_rc_t rc; + + switch (alg) { + case EFX_RX_HASHALG_LFSR: + EFX_RX_LFSR_HASH(enp, insert); + break; + + case EFX_RX_HASHALG_TOEPLITZ: + EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, + type & (1 << EFX_RX_HASH_IPV4), + type & (1 << EFX_RX_HASH_TCPIPV4)); + + EFX_RX_TOEPLITZ_IPV6_HASH(enp, + type & (1 << EFX_RX_HASH_IPV6), + type & (1 << EFX_RX_HASH_TCPIPV6), + rc); + if (rc != 0) + goto fail1; + + break; + + default: + rc = EINVAL; + goto fail2; + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + EFX_RX_LFSR_HASH(enp, B_FALSE); + + return (rc); +} +#endif + +#if EFSYS_OPT_RX_SCALE +static __checkReturn efx_rc_t +siena_rx_scale_key_set( + __in efx_nic_t *enp, + __in_ecount(n) uint8_t *key, + __in size_t n) +{ + efx_oword_t oword; + unsigned int byte; + unsigned int offset; + efx_rc_t rc; + + byte = 0; + + /* Write Toeplitz IPv4 hash key */ + EFX_ZERO_OWORD(oword); + for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; + offset > 0 && byte < n; + --offset) + oword.eo_u8[offset - 1] = key[byte++]; + + EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); + + byte = 0; + + /* Verify Toeplitz IPv4 hash key */ + EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); + for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; + offset > 0 && byte < n; + --offset) { + if (oword.eo_u8[offset - 1] != key[byte++]) { + rc = EFAULT; + goto fail1; + } + } + + if ((enp->en_features & EFX_FEATURE_IPV6) == 0) + goto done; + + byte = 0; + + /* Write Toeplitz IPv6 hash key 3 */ + EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); + for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + + FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; + offset > 0 && byte < n; + --offset) + oword.eo_u8[offset - 1] = key[byte++]; + + EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); + + /* Write Toeplitz IPv6 hash key 2 */ + EFX_ZERO_OWORD(oword); + for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + + FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; + offset > 0 && byte < n; + --offset) + oword.eo_u8[offset - 1] = key[byte++]; + + EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); + + /* Write Toeplitz IPv6 hash key 1 */ + EFX_ZERO_OWORD(oword); + for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + + FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; + offset > 0 && byte < n; + --offset) + oword.eo_u8[offset - 1] = key[byte++]; + + EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); + + byte = 0; + + /* Verify Toeplitz IPv6 hash key 3 */ + EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); + for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + + FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; + offset > 0 && byte < n; + --offset) { + if (oword.eo_u8[offset - 1] != key[byte++]) { + rc = EFAULT; + goto fail2; + } + } + + /* Verify Toeplitz IPv6 hash key 2 */ + EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); + for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + + FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; + offset > 0 && byte < n; + --offset) { + if (oword.eo_u8[offset - 1] != key[byte++]) { + rc = EFAULT; + goto fail3; + } + } + + /* Verify Toeplitz IPv6 hash key 1 */ + EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); + for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + + FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; + offset > 0 && byte < n; + --offset) { + if (oword.eo_u8[offset - 1] != key[byte++]) { + rc = EFAULT; + goto fail4; + } + } + +done: + return (0); + +fail4: + EFSYS_PROBE(fail4); +fail3: + EFSYS_PROBE(fail3); +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif + +#if EFSYS_OPT_RX_SCALE +static __checkReturn efx_rc_t +siena_rx_scale_tbl_set( + __in efx_nic_t *enp, + __in_ecount(n) unsigned int *table, + __in size_t n) +{ + efx_oword_t oword; + int index; + efx_rc_t rc; + + EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); + EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); + + if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { + rc = EINVAL; + goto fail1; + } + + for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { + uint32_t byte; + + /* Calculate the entry to place in the table */ + byte = (n > 0) ? (uint32_t)table[index % n] : 0; + + EFSYS_PROBE2(table, int, index, uint32_t, byte); + + EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); + + /* Write the table */ + EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, + index, &oword, B_TRUE); + } + + for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { + uint32_t byte; + + /* Determine if we're starting a new batch */ + byte = (n > 0) ? (uint32_t)table[index % n] : 0; + + /* Read the table */ + EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, + index, &oword, B_TRUE); + + /* Verify the entry */ + if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { + rc = EFAULT; + goto fail2; + } + } + + return (0); + +fail2: + EFSYS_PROBE(fail2); +fail1: + EFSYS_PROBE1(fail1, efx_rc_t, rc); + + return (rc); +} +#endif + /* * Falcon/Siena pseudo-header * -------------------------- @@ -531,6 +961,32 @@ fail1: * LL.LL LFSR hash (16-bit big-endian) */ +#if EFSYS_OPT_RX_SCALE +static __checkReturn uint32_t +siena_rx_prefix_hash( + __in efx_nic_t *enp, + __in efx_rx_hash_alg_t func, + __in uint8_t *buffer) +{ + _NOTE(ARGUNUSED(enp)) + + switch (func) { + case EFX_RX_HASHALG_TOEPLITZ: + return ((buffer[12] << 24) | + (buffer[13] << 16) | + (buffer[14] << 8) | + buffer[15]); + + case EFX_RX_HASHALG_LFSR: + return ((buffer[14] << 8) | buffer[15]); + + default: + EFSYS_ASSERT(0); + return (0); + } +} +#endif /* EFSYS_OPT_RX_SCALE */ + static __checkReturn efx_rc_t siena_rx_prefix_pktlen( __in efx_nic_t *enp, -- 2.20.1