common/mlx5: add glue functions on Windows
authorTal Shnaiderman <talshn@nvidia.com>
Mon, 28 Dec 2020 09:54:23 +0000 (11:54 +0200)
committerFerruh Yigit <ferruh.yigit@intel.com>
Fri, 8 Jan 2021 15:03:07 +0000 (16:03 +0100)
Windows glue functions are added to file mlx5/windows/mlx5_glue.c.
The following APIs are supported:
get_device_list, free_device_list, open_device, close_device,
query_device, query_hca_iseg, devx_obj_create, devx_obj_destroy,
devx_obj_query, devx_obj_modify, devx_general_cmd, devx_umem_reg,
devx_umem_dereg, devx_alloc_uar, devx_free_uar, devx_fs_rule_add,
devx_fs_rule_del, devx_query_eqn
New added files:
mlx5_win_defs.h - this file imports missing definitions from Linux
rdma-core library and Linux OS.
mlx5_win_ext.h - this file contains structs that enable a unified
Linux/Windows API. Each struct has an equivalent (but different) Linux
struct. By calling with 'void *' pointers - the Linux/Windows API is
identical.

Signed-off-by: Tal Shnaiderman <talshn@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
drivers/common/mlx5/rte_common_mlx5_exports.def
drivers/common/mlx5/windows/mlx5_glue.c [new file with mode: 0644]
drivers/common/mlx5/windows/mlx5_glue.h [new file with mode: 0644]
drivers/common/mlx5/windows/mlx5_win_defs.h [new file with mode: 0644]
drivers/common/mlx5/windows/mlx5_win_ext.h [new file with mode: 0644]

index 9e29f41..f8f930b 100644 (file)
@@ -36,6 +36,7 @@ EXPORTS
        mlx5_devx_cmd_create_flow_hit_aso_obj
 
        mlx5_get_dbr
+       mlx5_glue
 
        mlx5_malloc_mem_select
        mlx5_mr_btree_init
diff --git a/drivers/common/mlx5/windows/mlx5_glue.c b/drivers/common/mlx5/windows/mlx5_glue.c
new file mode 100644 (file)
index 0000000..7f8a00a
--- /dev/null
@@ -0,0 +1,304 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#include <errno.h>
+#include <stdalign.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <rte_malloc.h>
+
+#include "mlx5_glue.h"
+#include "mlx5_common_utils.h"
+#include "mlx5_win_ext.h"
+
+/*
+ * The returned value of this API is an array of pointers to mlx5
+ * devices under Windows. The interesting parameters of a device:
+ * Device PCI parameters: domain, bus, device id, function.
+ * Device name.
+ */
+static void *
+mlx5_glue_devx_get_device_list(int *num_devices)
+{
+       struct devx_device_bdf *devx_bdf_devs = NULL;
+       size_t n_devx_devx = 0;
+       int32_t ret = devx_get_device_list(&n_devx_devx, &devx_bdf_devs);
+
+       if (ret) {
+               errno = ret;
+               *num_devices = 0;
+               return NULL;
+       }
+       *num_devices = (int)n_devx_devx;
+       return devx_bdf_devs;
+}
+
+static void
+mlx5_glue_devx_free_device_list(void *list)
+{
+       if (!list) {
+               errno = EINVAL;
+               return;
+       }
+       devx_free_device_list(list);
+}
+
+static int
+mlx5_glue_devx_close_device(void *ctx)
+{
+       mlx5_context_st *mlx5_ctx;
+       int rc;
+
+       if (!ctx)
+               return -EINVAL;
+       mlx5_ctx = (mlx5_context_st *)ctx;
+       rc = devx_close_device(mlx5_ctx->devx_ctx);
+       free(mlx5_ctx);
+       return rc;
+}
+
+static void *
+mlx5_glue_devx_open_device(void *device)
+{
+       struct mlx5_context *mlx5_ctx;
+
+       if (!device) {
+               errno = EINVAL;
+               return NULL;
+       }
+       mlx5_ctx = malloc((sizeof(struct mlx5_context)));
+       if (!mlx5_ctx) {
+               errno = ENOMEM;
+               return NULL;
+       }
+       memset(mlx5_ctx, 0, sizeof(*mlx5_ctx));
+       mlx5_ctx->devx_ctx = devx_open_device(device);
+       if (DEVX_IS_ERR(mlx5_ctx->devx_ctx)) {
+               errno = -DEVX_PTR_ERR(mlx5_ctx->devx_ctx);
+               free(mlx5_ctx);
+               return NULL;
+       }
+       return mlx5_ctx;
+}
+
+static int
+mlx5_glue_devx_query_device(void *device_bdf, void *dev_inf)
+{
+       struct devx_device_bdf *dev_bdf;
+       struct devx_device *mlx5_dev;
+
+       if (!device_bdf)
+               return -EINVAL;
+       dev_bdf = (struct devx_device_bdf *)device_bdf;
+       mlx5_dev = (struct devx_device *)dev_inf;
+       int err = devx_query_device(dev_bdf, mlx5_dev);
+       if (err)
+               return -E_FAIL;
+       return 0;
+}
+
+static void *
+mlx5_glue_devx_query_hca_iseg_mapping(void *ctx, uint32_t *cb_iseg)
+{
+       struct mlx5_context *mlx5_ctx;
+       void *pv_iseg;
+       int err;
+
+       if (!ctx) {
+               errno = EINVAL;
+               return NULL;
+       }
+       mlx5_ctx = (struct mlx5_context *)ctx;
+       err = devx_query_hca_iseg_mapping(mlx5_ctx->devx_ctx,
+                                               cb_iseg, &pv_iseg);
+       if (err) {
+               errno = err;
+               return NULL;
+       }
+       return pv_iseg;
+}
+
+static void *
+mlx5_glue_devx_obj_create(void *ctx,
+                             void *in, size_t inlen,
+                             void *out, size_t outlen)
+{
+       mlx5_devx_obj_st *devx_obj;
+
+       if (!ctx) {
+               errno = EINVAL;
+               return NULL;
+       }
+       devx_obj = malloc((sizeof(*devx_obj)));
+       if (!devx_obj) {
+               errno = ENOMEM;
+               return NULL;
+       }
+       memset(devx_obj, 0, sizeof(*devx_obj));
+       devx_obj->devx_ctx = GET_DEVX_CTX(ctx);
+       devx_obj->obj = devx_obj_create(devx_obj->devx_ctx,
+                                       in, inlen, out, outlen);
+       if (DEVX_IS_ERR(devx_obj->obj)) {
+               errno = -DEVX_PTR_ERR(devx_obj->obj);
+               free(devx_obj);
+               return NULL;
+       }
+       return devx_obj;
+}
+
+static int
+mlx5_glue_devx_obj_destroy(void *obj)
+{
+       mlx5_devx_obj_st *devx_obj;
+
+       if (!obj)
+               return -EINVAL;
+       devx_obj = obj;
+       int rc = devx_obj_destroy(devx_obj->obj);
+       free(devx_obj);
+       return rc;
+}
+
+static int
+mlx5_glue_devx_general_cmd(void *ctx,
+                          void *in, size_t inlen,
+                          void *out, size_t outlen)
+{
+       if (!ctx)
+               return -EINVAL;
+       return devx_cmd(GET_DEVX_CTX(ctx), in, inlen, out, outlen);
+}
+
+static int
+mlx5_glue_devx_obj_query(void *obj,
+                           void *in, size_t inlen,
+                           void *out, size_t outlen)
+{
+       return devx_cmd(GET_OBJ_CTX(obj), in, inlen, out, outlen);
+}
+
+static int
+mlx5_glue_devx_obj_modify(void *obj,
+                           void *in, size_t inlen,
+                           void *out, size_t outlen)
+{
+       return devx_cmd(GET_OBJ_CTX(obj), in, inlen, out, outlen);
+}
+
+static int
+mlx5_glue_devx_umem_dereg(void *pumem)
+{
+       struct devx_obj_handle *umem;
+
+       if (!pumem)
+               return -EINVAL;
+       umem = pumem;
+       return devx_umem_unreg(umem);
+}
+
+static void *
+mlx5_glue_devx_umem_reg(void *ctx, void *addr, size_t size,
+                                 uint32_t access, uint32_t *id)
+{
+       struct devx_obj_handle *umem_hdl;
+       int w_access = DEVX_UMEM_ACCESS_READ;
+
+       if (!ctx) {
+               errno = EINVAL;
+               return NULL;
+       }
+       if (access)
+               w_access |= DEVX_UMEM_ACCESS_WRITE;
+
+       umem_hdl = devx_umem_reg(GET_DEVX_CTX(ctx), addr,
+                                       size, w_access, id);
+       if (DEVX_IS_ERR(umem_hdl)) {
+               errno = -DEVX_PTR_ERR(umem_hdl);
+               return NULL;
+       }
+       return umem_hdl;
+}
+
+static void *
+mlx5_glue_devx_alloc_uar(void *ctx,
+               uint32_t flags)
+{
+       devx_uar_handle *uar;
+
+       if (!ctx) {
+               errno = EINVAL;
+               return NULL;
+       }
+       uar = devx_alloc_uar(GET_DEVX_CTX(ctx), flags);
+       if (DEVX_IS_ERR(uar)) {
+               errno = -DEVX_PTR_ERR(uar);
+               return NULL;
+       }
+       return uar;
+}
+
+static int
+mlx5_glue_devx_query_eqn(void *ctx,
+               uint32_t cpus, uint32_t *eqn)
+{
+       if (!ctx)
+               return -EINVAL;
+       return devx_query_eqn(GET_DEVX_CTX(ctx), cpus, eqn);
+}
+
+static void
+mlx5_glue_devx_free_uar(void *uar)
+{
+       devx_free_uar((devx_uar_handle *)uar);
+}
+
+static void*
+mlx5_glue_devx_fs_rule_add(void *ctx, void *in, uint32_t inlen)
+
+{
+       struct devx_obj_handle *rule_hdl = NULL;
+
+       if (!ctx) {
+               errno = EINVAL;
+               return NULL;
+       }
+       rule_hdl = devx_fs_rule_add(GET_DEVX_CTX(ctx), in, inlen);
+       if (DEVX_IS_ERR(rule_hdl)) {
+               errno = -DEVX_PTR_ERR(rule_hdl);
+               return NULL;
+       }
+       return rule_hdl;
+}
+
+static int
+mlx5_glue_devx_fs_rule_del(void *flow)
+{
+       return devx_fs_rule_del(flow);
+}
+
+alignas(RTE_CACHE_LINE_SIZE)
+const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
+       .version = MLX5_GLUE_VERSION,
+       .get_device_list = mlx5_glue_devx_get_device_list,
+       .free_device_list = mlx5_glue_devx_free_device_list,
+       .open_device = mlx5_glue_devx_open_device,
+       .close_device = mlx5_glue_devx_close_device,
+       .query_device = mlx5_glue_devx_query_device,
+       .query_hca_iseg = mlx5_glue_devx_query_hca_iseg_mapping,
+       .devx_obj_create = mlx5_glue_devx_obj_create,
+       .devx_obj_destroy = mlx5_glue_devx_obj_destroy,
+       .devx_obj_query = mlx5_glue_devx_obj_query,
+       .devx_obj_modify = mlx5_glue_devx_obj_modify,
+       .devx_general_cmd = mlx5_glue_devx_general_cmd,
+       .devx_umem_reg = mlx5_glue_devx_umem_reg,
+       .devx_umem_dereg = mlx5_glue_devx_umem_dereg,
+       .devx_alloc_uar = mlx5_glue_devx_alloc_uar,
+       .devx_free_uar = mlx5_glue_devx_free_uar,
+       .devx_fs_rule_add = mlx5_glue_devx_fs_rule_add,
+       .devx_fs_rule_del = mlx5_glue_devx_fs_rule_del,
+       .devx_query_eqn = mlx5_glue_devx_query_eqn,
+};
diff --git a/drivers/common/mlx5/windows/mlx5_glue.h b/drivers/common/mlx5/windows/mlx5_glue.h
new file mode 100644 (file)
index 0000000..f2261ec
--- /dev/null
@@ -0,0 +1,58 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2020 Mellanox Technologies, Ltd
+ */
+
+#ifndef MLX5_GLUE_H_
+#define MLX5_GLUE_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <rte_byteorder.h>
+#include <mlx5_win_defs.h>
+
+#ifndef MLX5_GLUE_VERSION
+#define MLX5_GLUE_VERSION ""
+#endif
+
+/* LIB_GLUE_VERSION must be updated every time this structure is modified. */
+struct mlx5_glue {
+       const char *version;
+       void *(*devx_obj_create)(void *ctx,
+                                void *in, size_t inlen,
+                                void *out, size_t outlen);
+       int (*devx_obj_destroy)(void *obj);
+       int (*devx_obj_query)(void *obj,
+                             void *in, size_t inlen,
+                             void *out, size_t outlen);
+       int (*devx_obj_modify)(void *obj,
+                              void *in, size_t inlen,
+                              void *out, size_t outlen);
+       int (*devx_general_cmd)(void *ctx,
+                              void *in, size_t inlen,
+                              void *out, size_t outlen);
+       int (*devx_umem_dereg)(void *umem);
+       void *(*devx_umem_reg)(void *ctx,
+                       void *addr, size_t size,
+                       uint32_t access, uint32_t *id);
+       void *(*devx_alloc_uar)(void *ctx,
+                       uint32_t flags);
+       void (*devx_free_uar)(void *uar);
+       void *(*get_device_list)(int *num_devices);
+       void (*free_device_list)(void *list);
+       void *(*open_device)(void *device);
+       int (*close_device)(void *ctx);
+       int (*query_device)(void *device_bdf, void *dev_inf);
+       void* (*query_hca_iseg)(void *ctx, uint32_t *cb_iseg);
+       int (*devx_obj_query_async)(void *obj,
+                                   const void *in, size_t inlen,
+                                   size_t outlen, uint64_t wr_id,
+                                   void *cmd_comp);
+       void *(*devx_fs_rule_add)(void *ctx, void *in, uint32_t inlen);
+       int (*devx_fs_rule_del)(void *flow);
+       int (*devx_query_eqn)(void *context, uint32_t cpus, uint32_t *eqn);
+};
+
+extern const struct mlx5_glue *mlx5_glue;
+
+#endif /* MLX5_GLUE_H_ */
diff --git a/drivers/common/mlx5/windows/mlx5_win_defs.h b/drivers/common/mlx5/windows/mlx5_win_defs.h
new file mode 100644 (file)
index 0000000..72a3131
--- /dev/null
@@ -0,0 +1,25 @@
+/*
+ * Copyright (C) Mellanox Technologies, Ltd. 2001-2020.
+ *
+ */
+#ifndef __MLX5_WIN_DEFS_H__
+#define __MLX5_WIN_DEFS_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum {
+       MLX5_CQE_OWNER_MASK     = 1,
+       MLX5_CQE_REQ            = 0,
+       MLX5_CQE_RESP_WR_IMM    = 1,
+       MLX5_CQE_RESP_SEND      = 2,
+       MLX5_CQE_RESP_SEND_IMM  = 3,
+       MLX5_CQE_RESP_SEND_INV  = 4,
+       MLX5_CQE_RESIZE_CQ      = 5,
+       MLX5_CQE_NO_PACKET      = 6,
+       MLX5_CQE_REQ_ERR        = 13,
+       MLX5_CQE_RESP_ERR       = 14,
+       MLX5_CQE_INVALID        = 15,
+};
+#endif /* __MLX5_WIN_DEFS_H__ */
diff --git a/drivers/common/mlx5/windows/mlx5_win_ext.h b/drivers/common/mlx5/windows/mlx5_win_ext.h
new file mode 100644 (file)
index 0000000..0e74910
--- /dev/null
@@ -0,0 +1,34 @@
+/*
+ * Copyright (C) Mellanox Technologies, Ltd. 2001-2020.
+ *
+ */
+#ifndef __MLX5_WIN_ETX_H__
+#define __MLX5_WIN_ETX_H__
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "mlx5devx.h"
+
+typedef struct mlx5_context {
+       devx_device_ctx        *devx_ctx;
+       struct devx_device mlx5_dev;
+
+} mlx5_context_st;
+
+typedef struct {
+       devx_device_ctx *devx_ctx;
+       struct devx_obj_handle *obj;
+} mlx5_devx_obj_st;
+
+struct mlx5_devx_umem {
+       void                   *addr;
+       struct devx_obj_handle *umem_hdl;
+       uint32_t                umem_id;
+};
+
+#define GET_DEVX_CTX(ctx) (((mlx5_context_st *)ctx)->devx_ctx)
+#define GET_OBJ_CTX(obj)  (((mlx5_devx_obj_st *)obj)->devx_ctx)
+
+#endif /* __MLX5_WIN_ETX_H__ */