+ ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge);
+ if (ret < 0) {
+ TESTPMD_LOG(ERR, "Cannot create memory area\n");
+ return -1;
+ }
+
+ /* we now have a valid memory area, so add it to heap */
+ ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME,
+ param.addr, param.len, param.iova_table,
+ param.iova_table_len, param.pgsz);
+
+ /* when using VFIO, memory is automatically mapped for DMA by EAL */
+
+ /* not needed any more */
+ free(param.iova_table);
+
+ if (ret < 0) {
+ TESTPMD_LOG(ERR, "Cannot add memory to heap\n");
+ munmap(param.addr, param.len);
+ return -1;
+ }
+
+ /* success */
+
+ TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n",
+ param.len >> 20);
+
+ return 0;
+}
+static void
+dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
+ struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
+{
+ uint16_t pid = 0;
+ int ret;
+
+ RTE_ETH_FOREACH_DEV(pid) {
+ struct rte_eth_dev_info dev_info;
+
+ ret = eth_dev_info_get_print_err(pid, &dev_info);
+ if (ret != 0) {
+ TESTPMD_LOG(DEBUG,
+ "unable to get device info for port %d on addr 0x%p,"
+ "mempool unmapping will not be performed\n",
+ pid, memhdr->addr);
+ continue;
+ }
+
+ ret = rte_dev_dma_unmap(dev_info.device, memhdr->addr, 0, memhdr->len);
+ if (ret) {
+ TESTPMD_LOG(DEBUG,
+ "unable to DMA unmap addr 0x%p "
+ "for device %s\n",
+ memhdr->addr, dev_info.device->name);
+ }
+ }
+ ret = rte_extmem_unregister(memhdr->addr, memhdr->len);
+ if (ret) {
+ TESTPMD_LOG(DEBUG,
+ "unable to un-register addr 0x%p\n", memhdr->addr);
+ }
+}
+
+static void
+dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused,
+ struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused)
+{
+ uint16_t pid = 0;
+ size_t page_size = sysconf(_SC_PAGESIZE);
+ int ret;
+
+ ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0,
+ page_size);
+ if (ret) {
+ TESTPMD_LOG(DEBUG,
+ "unable to register addr 0x%p\n", memhdr->addr);
+ return;
+ }
+ RTE_ETH_FOREACH_DEV(pid) {
+ struct rte_eth_dev_info dev_info;
+
+ ret = eth_dev_info_get_print_err(pid, &dev_info);
+ if (ret != 0) {
+ TESTPMD_LOG(DEBUG,
+ "unable to get device info for port %d on addr 0x%p,"
+ "mempool mapping will not be performed\n",
+ pid, memhdr->addr);
+ continue;
+ }
+ ret = rte_dev_dma_map(dev_info.device, memhdr->addr, 0, memhdr->len);
+ if (ret) {
+ TESTPMD_LOG(DEBUG,
+ "unable to DMA map addr 0x%p "
+ "for device %s\n",
+ memhdr->addr, dev_info.device->name);
+ }
+ }
+}
+#endif
+
+static unsigned int
+setup_extbuf(uint32_t nb_mbufs, uint16_t mbuf_sz, unsigned int socket_id,
+ char *pool_name, struct rte_pktmbuf_extmem **ext_mem)
+{
+ struct rte_pktmbuf_extmem *xmem;
+ unsigned int ext_num, zone_num, elt_num;
+ uint16_t elt_size;
+
+ elt_size = RTE_ALIGN_CEIL(mbuf_sz, RTE_CACHE_LINE_SIZE);
+ elt_num = EXTBUF_ZONE_SIZE / elt_size;
+ zone_num = (nb_mbufs + elt_num - 1) / elt_num;
+
+ xmem = malloc(sizeof(struct rte_pktmbuf_extmem) * zone_num);
+ if (xmem == NULL) {
+ TESTPMD_LOG(ERR, "Cannot allocate memory for "
+ "external buffer descriptors\n");
+ *ext_mem = NULL;
+ return 0;
+ }
+ for (ext_num = 0; ext_num < zone_num; ext_num++) {
+ struct rte_pktmbuf_extmem *xseg = xmem + ext_num;
+ const struct rte_memzone *mz;
+ char mz_name[RTE_MEMZONE_NAMESIZE];
+ int ret;
+
+ ret = snprintf(mz_name, sizeof(mz_name),
+ RTE_MEMPOOL_MZ_FORMAT "_xb_%u", pool_name, ext_num);
+ if (ret < 0 || ret >= (int)sizeof(mz_name)) {
+ errno = ENAMETOOLONG;
+ ext_num = 0;
+ break;
+ }
+ mz = rte_memzone_reserve_aligned(mz_name, EXTBUF_ZONE_SIZE,
+ socket_id,
+ RTE_MEMZONE_IOVA_CONTIG |
+ RTE_MEMZONE_1GB |
+ RTE_MEMZONE_SIZE_HINT_ONLY,
+ EXTBUF_ZONE_SIZE);
+ if (mz == NULL) {
+ /*
+ * The caller exits on external buffer creation
+ * error, so there is no need to free memzones.
+ */
+ errno = ENOMEM;
+ ext_num = 0;
+ break;
+ }
+ xseg->buf_ptr = mz->addr;
+ xseg->buf_iova = mz->iova;
+ xseg->buf_len = EXTBUF_ZONE_SIZE;
+ xseg->elt_size = elt_size;
+ }
+ if (ext_num == 0 && xmem != NULL) {
+ free(xmem);
+ xmem = NULL;
+ }
+ *ext_mem = xmem;
+ return ext_num;
+}
+
+/*
+ * Configuration initialisation done once at init time.
+ */
+static struct rte_mempool *
+mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf,
+ unsigned int socket_id, uint16_t size_idx)
+{
+ char pool_name[RTE_MEMPOOL_NAMESIZE];
+ struct rte_mempool *rte_mp = NULL;
+#ifndef RTE_EXEC_ENV_WINDOWS
+ uint32_t mb_size;
+
+ mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size;
+#endif
+ mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name), size_idx);
+ if (!is_proc_primary()) {
+ rte_mp = rte_mempool_lookup(pool_name);
+ if (rte_mp == NULL)
+ rte_exit(EXIT_FAILURE,
+ "Get mbuf pool for socket %u failed: %s\n",
+ socket_id, rte_strerror(rte_errno));
+ return rte_mp;
+ }
+
+ TESTPMD_LOG(INFO,
+ "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n",
+ pool_name, nb_mbuf, mbuf_seg_size, socket_id);
+
+ switch (mp_alloc_type) {
+ case MP_ALLOC_NATIVE:
+ {
+ /* wrapper to rte_mempool_create() */
+ TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
+ rte_mbuf_best_mempool_ops());
+ rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
+ mb_mempool_cache, 0, mbuf_seg_size, socket_id);
+ break;
+ }
+#ifndef RTE_EXEC_ENV_WINDOWS
+ case MP_ALLOC_ANON:
+ {
+ rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf,
+ mb_size, (unsigned int) mb_mempool_cache,
+ sizeof(struct rte_pktmbuf_pool_private),
+ socket_id, mempool_flags);
+ if (rte_mp == NULL)
+ goto err;
+
+ if (rte_mempool_populate_anon(rte_mp) == 0) {
+ rte_mempool_free(rte_mp);
+ rte_mp = NULL;
+ goto err;
+ }
+ rte_pktmbuf_pool_init(rte_mp, NULL);
+ rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL);
+ rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL);
+ break;
+ }
+ case MP_ALLOC_XMEM:
+ case MP_ALLOC_XMEM_HUGE:
+ {
+ int heap_socket;
+ bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE;
+
+ if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0)
+ rte_exit(EXIT_FAILURE, "Could not create external memory\n");
+
+ heap_socket =
+ rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME);
+ if (heap_socket < 0)
+ rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n");
+
+ TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
+ rte_mbuf_best_mempool_ops());
+ rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf,
+ mb_mempool_cache, 0, mbuf_seg_size,
+ heap_socket);
+ break;
+ }
+#endif
+ case MP_ALLOC_XBUF:
+ {
+ struct rte_pktmbuf_extmem *ext_mem;
+ unsigned int ext_num;
+
+ ext_num = setup_extbuf(nb_mbuf, mbuf_seg_size,
+ socket_id, pool_name, &ext_mem);
+ if (ext_num == 0)
+ rte_exit(EXIT_FAILURE,
+ "Can't create pinned data buffers\n");
+
+ TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n",
+ rte_mbuf_best_mempool_ops());
+ rte_mp = rte_pktmbuf_pool_create_extbuf
+ (pool_name, nb_mbuf, mb_mempool_cache,
+ 0, mbuf_seg_size, socket_id,
+ ext_mem, ext_num);
+ free(ext_mem);
+ break;
+ }
+ default:
+ {
+ rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n");
+ }
+ }
+
+#ifndef RTE_EXEC_ENV_WINDOWS
+err:
+#endif
+ if (rte_mp == NULL) {
+ rte_exit(EXIT_FAILURE,
+ "Creation of mbuf pool for socket %u failed: %s\n",
+ socket_id, rte_strerror(rte_errno));
+ } else if (verbose_level > 0) {
+ rte_mempool_dump(stdout, rte_mp);
+ }
+ return rte_mp;
+}
+
+/*
+ * Check given socket id is valid or not with NUMA mode,
+ * if valid, return 0, else return -1
+ */
+static int
+check_socket_id(const unsigned int socket_id)
+{
+ static int warning_once = 0;
+
+ if (new_socket_id(socket_id)) {
+ if (!warning_once && numa_support)
+ fprintf(stderr,
+ "Warning: NUMA should be configured manually by using --port-numa-config and --ring-numa-config parameters along with --numa.\n");
+ warning_once = 1;
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * Get the allowed maximum number of RX queues.
+ * *pid return the port id which has minimal value of
+ * max_rx_queues in all ports.
+ */
+queueid_t
+get_allowed_max_nb_rxq(portid_t *pid)
+{
+ queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT;
+ bool max_rxq_valid = false;
+ portid_t pi;
+ struct rte_eth_dev_info dev_info;
+
+ RTE_ETH_FOREACH_DEV(pi) {
+ if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
+ continue;
+
+ max_rxq_valid = true;
+ if (dev_info.max_rx_queues < allowed_max_rxq) {
+ allowed_max_rxq = dev_info.max_rx_queues;
+ *pid = pi;
+ }
+ }
+ return max_rxq_valid ? allowed_max_rxq : 0;
+}
+
+/*
+ * Check input rxq is valid or not.
+ * If input rxq is not greater than any of maximum number
+ * of RX queues of all ports, it is valid.
+ * if valid, return 0, else return -1
+ */
+int
+check_nb_rxq(queueid_t rxq)
+{
+ queueid_t allowed_max_rxq;
+ portid_t pid = 0;
+
+ allowed_max_rxq = get_allowed_max_nb_rxq(&pid);
+ if (rxq > allowed_max_rxq) {
+ fprintf(stderr,
+ "Fail: input rxq (%u) can't be greater than max_rx_queues (%u) of port %u\n",
+ rxq, allowed_max_rxq, pid);
+ return -1;
+ }
+ return 0;
+}
+
+/*
+ * Get the allowed maximum number of TX queues.
+ * *pid return the port id which has minimal value of
+ * max_tx_queues in all ports.
+ */
+queueid_t
+get_allowed_max_nb_txq(portid_t *pid)
+{
+ queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT;
+ bool max_txq_valid = false;
+ portid_t pi;
+ struct rte_eth_dev_info dev_info;
+
+ RTE_ETH_FOREACH_DEV(pi) {
+ if (eth_dev_info_get_print_err(pi, &dev_info) != 0)
+ continue;
+
+ max_txq_valid = true;