From 56270b4208ab7b6881ac9d8ed4ebb5bc6529715a Mon Sep 17 00:00:00 2001 From: Alvin Zhang Date: Tue, 24 Sep 2019 22:24:43 +0800 Subject: [PATCH] net/i40e: limit the number of VF messages PF driver supports counting VF adminQ messages. If any VF driver sends much more adminQ messages to its PF driver in a period of time, it will trigger the PF's message limitation, then in the next certain amount of seconds the PF driver will ignore any new adminQ message from that VF. Signed-off-by: Alvin Zhang Reviewed-by: Xiaolong Ye Acked-by: Qi Zhang --- doc/guides/nics/i40e.rst | 10 +++++ drivers/net/i40e/i40e_ethdev.c | 70 ++++++++++++++++++++++++++++++++++ drivers/net/i40e/i40e_ethdev.h | 32 ++++++++++++++++ drivers/net/i40e/i40e_pf.c | 65 ++++++++++++++++++++++++++++++- 4 files changed, 175 insertions(+), 2 deletions(-) diff --git a/doc/guides/nics/i40e.rst b/doc/guides/nics/i40e.rst index 0884e15540..38acf5906d 100644 --- a/doc/guides/nics/i40e.rst +++ b/doc/guides/nics/i40e.rst @@ -185,6 +185,16 @@ Runtime Config Options -w 84:00.0,use-latest-supported-vec=1 +- ``Enable validation for VF message`` (default ``not enabled``) + + The PF counts messages from each VF. If in any period of seconds the message + statistic from a VF exceeds maximal limitation, the PF will ignore any new message + from that VF for some seconds. + Format -- "maximal-message@period-seconds:ignore-seconds" + For example:: + + -w 84:00.0,vf_msg_cfg=80@120:180 + Vector RX Pre-conditions ~~~~~~~~~~~~~~~~~~~~~~~~ For Vector RX it is assumed that the number of descriptor rings will be a power diff --git a/drivers/net/i40e/i40e_ethdev.c b/drivers/net/i40e/i40e_ethdev.c index 93852c15d5..76cd9f510a 100644 --- a/drivers/net/i40e/i40e_ethdev.c +++ b/drivers/net/i40e/i40e_ethdev.c @@ -44,6 +44,7 @@ #define ETH_I40E_SUPPORT_MULTI_DRIVER "support-multi-driver" #define ETH_I40E_QUEUE_NUM_PER_VF_ARG "queue-num-per-vf" #define ETH_I40E_USE_LATEST_VEC "use-latest-supported-vec" +#define ETH_I40E_VF_MSG_CFG "vf_msg_cfg" #define I40E_CLEAR_PXE_WAIT_MS 200 @@ -406,6 +407,7 @@ static const char *const valid_keys[] = { ETH_I40E_SUPPORT_MULTI_DRIVER, ETH_I40E_QUEUE_NUM_PER_VF_ARG, ETH_I40E_USE_LATEST_VEC, + ETH_I40E_VF_MSG_CFG, NULL}; static const struct rte_pci_id pci_id_i40e_map[] = { @@ -1256,6 +1258,73 @@ i40e_use_latest_vec(struct rte_eth_dev *dev) return 0; } +static int +read_vf_msg_config(__rte_unused const char *key, + const char *value, + void *opaque) +{ + struct i40e_vf_msg_cfg *cfg = opaque; + + if (sscanf(value, "%u@%u:%u", &cfg->max_msg, &cfg->period, + &cfg->ignore_second) != 3) { + memset(cfg, 0, sizeof(*cfg)); + PMD_DRV_LOG(ERR, "format error! example: " + "%s=60@120:180", ETH_I40E_VF_MSG_CFG); + return -EINVAL; + } + + /* + * If the message validation function been enabled, the 'period' + * and 'ignore_second' must greater than 0. + */ + if (cfg->max_msg && (!cfg->period || !cfg->ignore_second)) { + memset(cfg, 0, sizeof(*cfg)); + PMD_DRV_LOG(ERR, "%s error! the second and third" + " number must be greater than 0!", + ETH_I40E_VF_MSG_CFG); + return -EINVAL; + } + + return 0; +} + +static int +i40e_parse_vf_msg_config(struct rte_eth_dev *dev, + struct i40e_vf_msg_cfg *msg_cfg) +{ + struct rte_kvargs *kvlist; + int kvargs_count; + int ret = 0; + + memset(msg_cfg, 0, sizeof(*msg_cfg)); + + if (!dev->device->devargs) + return ret; + + kvlist = rte_kvargs_parse(dev->device->devargs->args, valid_keys); + if (!kvlist) + return -EINVAL; + + kvargs_count = rte_kvargs_count(kvlist, ETH_I40E_VF_MSG_CFG); + if (!kvargs_count) + goto free_end; + + if (kvargs_count > 1) { + PMD_DRV_LOG(ERR, "More than one argument \"%s\"!", + ETH_I40E_VF_MSG_CFG); + ret = -EINVAL; + goto free_end; + } + + if (rte_kvargs_process(kvlist, ETH_I40E_VF_MSG_CFG, + read_vf_msg_config, msg_cfg) < 0) + ret = -EINVAL; + +free_end: + rte_kvargs_free(kvlist); + return ret; +} + #define I40E_ALARM_INTERVAL 50000 /* us */ static int @@ -1328,6 +1397,7 @@ eth_i40e_dev_init(struct rte_eth_dev *dev, void *init_params __rte_unused) return -EIO; } + i40e_parse_vf_msg_config(dev, &pf->vf_msg_cfg); /* Check if need to support multi-driver */ i40e_support_multi_driver(dev); /* Check if users want the latest supported vec path */ diff --git a/drivers/net/i40e/i40e_ethdev.h b/drivers/net/i40e/i40e_ethdev.h index 38ac3ead67..261954b9a0 100644 --- a/drivers/net/i40e/i40e_ethdev.h +++ b/drivers/net/i40e/i40e_ethdev.h @@ -426,6 +426,22 @@ struct i40e_pf_vf { /* version of the virtchnl from VF */ struct virtchnl_version_info version; uint32_t request_caps; /* offload caps requested from VF */ + + /* + * Variables for store the arrival timestamp of VF messages. + * If the timestamp of latest message stored at + * `msg_timestamps[index % max]` then the timestamp of + * earliest message stored at `msg_time[(index + 1) % max]`. + * When a new message come, the timestamp of this message + * will be stored at `msg_timestamps[(index + 1) % max]` and the + * earliest message timestamp is at + * `msg_timestamps[(index + 2) % max]` now... + */ + uint32_t msg_index; + uint64_t *msg_timestamps; + + /* cycle of stop ignoring VF message */ + uint64_t ignore_end_cycle; }; /* @@ -900,6 +916,20 @@ struct i40e_rte_flow_rss_conf { uint16_t queue[I40E_MAX_Q_PER_TC]; /**< Queues indices to use. */ }; +struct i40e_vf_msg_cfg { + /* maximal VF message during a statistic period */ + uint32_t max_msg; + + /* statistic period, in second */ + uint32_t period; + /* + * If message statistics from a VF exceed the maximal limitation, + * the PF will ignore any new message from that VF for + * 'ignor_second' time. + */ + uint32_t ignore_second; +}; + /* * Structure to store private data specific for PF instance. */ @@ -975,6 +1005,8 @@ struct i40e_pf { struct i40e_customized_pctype customized_pctype[I40E_CUSTOMIZED_MAX]; /* Switch Domain Id */ uint16_t switch_domain_id; + + struct i40e_vf_msg_cfg vf_msg_cfg; }; enum pending_msg { diff --git a/drivers/net/i40e/i40e_pf.c b/drivers/net/i40e/i40e_pf.c index b28d02ec2a..7bf1e79410 100644 --- a/drivers/net/i40e/i40e_pf.c +++ b/drivers/net/i40e/i40e_pf.c @@ -1297,6 +1297,7 @@ i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, /* AdminQ will pass absolute VF id, transfer to internal vf id */ uint16_t vf_id = abs_vf_id - hw->func_caps.vf_base_id; struct rte_pmd_i40e_mb_event_param ret_param; + uint64_t first_cycle, cur_cycle; bool b_op = TRUE; int ret; @@ -1306,11 +1307,18 @@ i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, } vf = &pf->vfs[vf_id]; + + cur_cycle = rte_get_timer_cycles(); + + /* if the VF being blocked, ignore the message and return */ + if (cur_cycle < vf->ignore_end_cycle) + return; + if (!vf->vsi) { PMD_DRV_LOG(ERR, "NO VSI associated with VF found"); i40e_pf_host_send_msg_to_vf(vf, opcode, I40E_ERR_NO_AVAILABLE_VSI, NULL, 0); - return; + goto check; } /* perform basic checks on the msg */ @@ -1334,7 +1342,7 @@ i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, vf_id, opcode, msglen); i40e_pf_host_send_msg_to_vf(vf, opcode, I40E_ERR_PARAM, NULL, 0); - return; + goto check; } /** @@ -1456,6 +1464,37 @@ i40e_pf_host_handle_vf_msg(struct rte_eth_dev *dev, NULL, 0); break; } + +check: + /* if message validation not enabled */ + if (!pf->vf_msg_cfg.max_msg) + return; + + /* store current cycle */ + vf->msg_timestamps[vf->msg_index++] = cur_cycle; + vf->msg_index %= pf->vf_msg_cfg.max_msg; + + /* read the timestamp of earliest message */ + first_cycle = vf->msg_timestamps[vf->msg_index]; + + /* + * If the time span from the arrival time of first message to + * the arrival time of current message smaller than `period`, + * that mean too much message in this statistic period. + */ + if (first_cycle && cur_cycle < first_cycle + + (uint64_t)pf->vf_msg_cfg.period * rte_get_timer_hz()) { + PMD_DRV_LOG(WARNING, "VF %u too much messages(%u in %u" + " seconds),\n\tany new message from which" + " will be ignored during next %u seconds!", + vf_id, pf->vf_msg_cfg.max_msg, + (uint32_t)((cur_cycle - first_cycle + + rte_get_timer_hz() - 1) / rte_get_timer_hz()), + pf->vf_msg_cfg.ignore_second); + vf->ignore_end_cycle = rte_get_timer_cycles() + + pf->vf_msg_cfg.ignore_second * + rte_get_timer_hz(); + } } int @@ -1463,6 +1502,7 @@ i40e_pf_host_init(struct rte_eth_dev *dev) { struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_PF_TO_HW(pf); + size_t size; int ret, i; uint32_t val; @@ -1489,10 +1529,24 @@ i40e_pf_host_init(struct rte_eth_dev *dev) I40E_WRITE_REG(hw, I40E_PFGEN_PORTMDIO_NUM, val); I40E_WRITE_FLUSH(hw); + /* calculate the memory size for storing timestamp of messages */ + size = pf->vf_msg_cfg.max_msg * sizeof(uint64_t); + for (i = 0; i < pf->vf_num; i++) { pf->vfs[i].pf = pf; pf->vfs[i].state = I40E_VF_INACTIVE; pf->vfs[i].vf_idx = i; + + if (size) { + /* allocate memory for store timestamp of messages */ + pf->vfs[i].msg_timestamps = + rte_zmalloc("i40e_pf_vf", size, 0); + if (pf->vfs[i].msg_timestamps == NULL) { + ret = -ENOMEM; + goto fail; + } + } + ret = i40e_pf_host_vf_reset(&pf->vfs[i], 0); if (ret != I40E_SUCCESS) goto fail; @@ -1505,6 +1559,8 @@ i40e_pf_host_init(struct rte_eth_dev *dev) return I40E_SUCCESS; fail: + for (; i >= 0; i--) + rte_free(pf->vfs[i].msg_timestamps); rte_free(pf->vfs); i40e_pf_enable_irq0(hw); @@ -1517,6 +1573,7 @@ i40e_pf_host_uninit(struct rte_eth_dev *dev) struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private); struct i40e_hw *hw = I40E_PF_TO_HW(pf); uint32_t val; + int i; PMD_INIT_FUNC_TRACE(); @@ -1529,6 +1586,10 @@ i40e_pf_host_uninit(struct rte_eth_dev *dev) (pf->vf_nb_qps == 0)) return I40E_SUCCESS; + /* free memory for store timestamp of messages */ + for (i = 0; i < pf->vf_num; i++) + rte_free(pf->vfs[i].msg_timestamps); + /* free memory to store VF structure */ rte_free(pf->vfs); pf->vfs = NULL; -- 2.20.1