vdpa/mlx5: fix large VM memory region registration

When VM size is larger than 4G (u32) and memory region is larger than 4G,
the 32-bit GCD function overflowed and returned wrong value
that resulted in memory registration failure.

This patch calls 64-bit GCD function to avoid overflow.

Fixes: cc07a42da250 ("vdpa/mlx5: prepare memory regions")
Cc: stable@dpdk.org
Signed-off-by: Xueming Li <xuemingl@nvidia.com>
Reviewed-by: Matan Azrad <matan@nvidia.com>

diff --git a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
index 59ce4e8..a06681b 100644 (file)
--- a/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
+++ b/drivers/vdpa/mlx5/mlx5_vdpa_mem.c
@@ -103,15 +103,15 @@ mlx5_vdpa_vhost_mem_regions_prepare(int vid, uint8_t *mode, uint64_t *mem_size,
                        size = mem->regions[i].guest_phys_addr -
                                (mem->regions[i - 1].guest_phys_addr +
                                 mem->regions[i - 1].size);
-                       *gcd = rte_get_gcd(*gcd, size);
+                       *gcd = rte_get_gcd64(*gcd, size);
                        klm_entries_num += KLM_NUM_MAX_ALIGN(size);
                }
                size = mem->regions[i].size;
-               *gcd = rte_get_gcd(*gcd, size);
+               *gcd = rte_get_gcd64(*gcd, size);
                klm_entries_num += KLM_NUM_MAX_ALIGN(size);
        }
        if (*gcd > MLX5_MAX_KLM_BYTE_COUNT)
-               *gcd = rte_get_gcd(*gcd, MLX5_MAX_KLM_BYTE_COUNT);
+               *gcd = rte_get_gcd64(*gcd, MLX5_MAX_KLM_BYTE_COUNT);
        if (!RTE_IS_POWER_OF_2(*gcd)) {
                uint64_t candidate_gcd = rte_align64prevpow2(*gcd);