From 6a113992060e2f59590fc6d296ccadc102a56c23 Mon Sep 17 00:00:00 2001 From: Shrikrishna Khare Date: Mon, 6 Mar 2017 14:55:06 -0800 Subject: [PATCH] net/vmxnet3: add cmd to register memory region In vmxnet3 version 3, the emulation added support for the vmxnet3 driver to communicate information about the memory regions the driver will use for rx/tx buffers. The driver can also indicate which rx/tx queue the memory region is applicable for. If this information is communicated to the emulation, the emulation will always keep these memory regions mapped, thereby avoiding the mapping/unmapping overhead for every packet. Signed-off-by: Shrikrishna Khare Signed-off-by: Guolin Yang Acked-by: Yong Wang Acked-by: Jin Heo --- drivers/net/vmxnet3/base/vmxnet3_defs.h | 25 ++++++ drivers/net/vmxnet3/vmxnet3_ethdev.c | 102 ++++++++++++++++++++++++ drivers/net/vmxnet3/vmxnet3_ethdev.h | 2 + 3 files changed, 129 insertions(+) diff --git a/drivers/net/vmxnet3/base/vmxnet3_defs.h b/drivers/net/vmxnet3/base/vmxnet3_defs.h index c708498057..bfa9622d0b 100644 --- a/drivers/net/vmxnet3/base/vmxnet3_defs.h +++ b/drivers/net/vmxnet3/base/vmxnet3_defs.h @@ -111,6 +111,7 @@ typedef enum { VMXNET3_CMD_ACTIVATE_VF, VMXNET3_CMD_RESERVED3, VMXNET3_CMD_RESERVED4, + VMXNET3_CMD_REGISTER_MEMREGS, VMXNET3_CMD_FIRST_GET = 0xF00D0000, VMXNET3_CMD_GET_QUEUE_STATUS = VMXNET3_CMD_FIRST_GET, @@ -722,6 +723,30 @@ struct Vmxnet3_SetPolling { #include "vmware_pack_end.h" Vmxnet3_SetPolling; +typedef +#include "vmware_pack_begin.h" +struct Vmxnet3_MemoryRegion { + __le64 startPA; + __le32 length; + __le16 txQueueBits; /* bit n corresponding to tx queue n */ + __le16 rxQueueBits; /* bit n corresponding to rx queue n */ +} +#include "vmware_pack_end.h" +Vmxnet3_MemoryRegion; + +#define MAX_MEMORY_REGION_PER_QUEUE 16 +#define MAX_MEMORY_REGION_PER_DEVICE 256 + +typedef +#include "vmware_pack_begin.h" +struct Vmxnet3_MemRegs { + __le16 numRegs; + __le16 pad[3]; + Vmxnet3_MemoryRegion memRegs[1]; +} +#include "vmware_pack_end.h" +Vmxnet3_MemRegs; + /* * If the command data <= 16 bytes, use the shared memory direcly. * Otherwise, use the variable length configuration descriptor. diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.c b/drivers/net/vmxnet3/vmxnet3_ethdev.c index d92d9ccc78..bb22b6d23b 100644 --- a/drivers/net/vmxnet3/vmxnet3_ethdev.c +++ b/drivers/net/vmxnet3/vmxnet3_ethdev.c @@ -488,6 +488,92 @@ vmxnet3_write_mac(struct vmxnet3_hw *hw, const uint8_t *addr) VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_MACH, val); } +static int +vmxnet3_dev_setup_memreg(struct rte_eth_dev *dev) +{ + struct vmxnet3_hw *hw = dev->data->dev_private; + Vmxnet3_DriverShared *shared = hw->shared; + Vmxnet3_CmdInfo *cmdInfo; + struct rte_mempool *mp[VMXNET3_MAX_RX_QUEUES]; + uint8_t index[VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES]; + uint32_t num, i, j, size; + + if (hw->memRegsPA == 0) { + const struct rte_memzone *mz; + + size = sizeof(Vmxnet3_MemRegs) + + (VMXNET3_MAX_RX_QUEUES + VMXNET3_MAX_TX_QUEUES) * + sizeof(Vmxnet3_MemoryRegion); + + mz = gpa_zone_reserve(dev, size, "memRegs", rte_socket_id(), 8, + 1); + if (mz == NULL) { + PMD_INIT_LOG(ERR, "ERROR: Creating memRegs zone"); + return -ENOMEM; + } + memset(mz->addr, 0, mz->len); + hw->memRegs = mz->addr; + hw->memRegsPA = mz->phys_addr; + } + + num = hw->num_rx_queues; + + for (i = 0; i < num; i++) { + vmxnet3_rx_queue_t *rxq = dev->data->rx_queues[i]; + + mp[i] = rxq->mp; + index[i] = 1 << i; + } + + /* + * The same mempool could be used by multiple queues. In such a case, + * remove duplicate mempool entries. Only one entry is kept with + * bitmask indicating queues that are using this mempool. + */ + for (i = 1; i < num; i++) { + for (j = 0; j < i; j++) { + if (mp[i] == mp[j]) { + mp[i] = NULL; + index[j] |= 1 << i; + break; + } + } + } + + j = 0; + for (i = 0; i < num; i++) { + if (mp[i] == NULL) + continue; + + Vmxnet3_MemoryRegion *mr = &hw->memRegs->memRegs[j]; + + mr->startPA = + (uintptr_t)STAILQ_FIRST(&mp[i]->mem_list)->phys_addr; + mr->length = STAILQ_FIRST(&mp[i]->mem_list)->len <= INT32_MAX ? + STAILQ_FIRST(&mp[i]->mem_list)->len : INT32_MAX; + mr->txQueueBits = index[i]; + mr->rxQueueBits = index[i]; + + PMD_INIT_LOG(INFO, + "index: %u startPA: %" PRIu64 " length: %u, " + "rxBits: %x", + j, mr->startPA, mr->length, mr->rxQueueBits); + j++; + } + hw->memRegs->numRegs = j; + PMD_INIT_LOG(INFO, "numRegs: %u", j); + + size = sizeof(Vmxnet3_MemRegs) + + (j - 1) * sizeof(Vmxnet3_MemoryRegion); + + cmdInfo = &shared->cu.cmdInfo; + cmdInfo->varConf.confVer = 1; + cmdInfo->varConf.confLen = size; + cmdInfo->varConf.confPA = hw->memRegsPA; + + return 0; +} + static int vmxnet3_setup_driver_shared(struct rte_eth_dev *dev) { @@ -628,6 +714,20 @@ vmxnet3_dev_start(struct rte_eth_dev *dev) return -EINVAL; } + /* Setup memory region for rx buffers */ + ret = vmxnet3_dev_setup_memreg(dev); + if (ret == 0) { + VMXNET3_WRITE_BAR1_REG(hw, VMXNET3_REG_CMD, + VMXNET3_CMD_REGISTER_MEMREGS); + ret = VMXNET3_READ_BAR1_REG(hw, VMXNET3_REG_CMD); + if (ret != 0) + PMD_INIT_LOG(DEBUG, + "Failed in setup memory region cmd\n"); + ret = 0; + } else { + PMD_INIT_LOG(DEBUG, "Failed to setup memory region\n"); + } + /* Disable interrupts */ vmxnet3_disable_intr(hw); @@ -641,6 +741,8 @@ vmxnet3_dev_start(struct rte_eth_dev *dev) return ret; } + hw->adapter_stopped = FALSE; + /* Setting proper Rx Mode and issue Rx Mode Update command */ vmxnet3_dev_set_rxmode(hw, VMXNET3_RXM_UCAST | VMXNET3_RXM_BCAST, 1); diff --git a/drivers/net/vmxnet3/vmxnet3_ethdev.h b/drivers/net/vmxnet3/vmxnet3_ethdev.h index 9d00cd101f..7a03262978 100644 --- a/drivers/net/vmxnet3/vmxnet3_ethdev.h +++ b/drivers/net/vmxnet3/vmxnet3_ethdev.h @@ -119,6 +119,8 @@ struct vmxnet3_hw { uint64_t rss_confPA; vmxnet3_mf_table_t *mf_table; uint32_t shadow_vfta[VMXNET3_VFT_SIZE]; + Vmxnet3_MemRegs *memRegs; + uint64_t memRegsPA; #define VMXNET3_VFT_TABLE_SIZE (VMXNET3_VFT_SIZE * sizeof(uint32_t)) }; -- 2.20.1