1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2018 6WIND S.A.
3 * Copyright 2018 Mellanox Technologies, Ltd
13 * Not needed by this file; included to work around the lack of off_t
14 * definition for mlx5dv.h with unpatched rdma-core versions.
16 #include <sys/types.h>
18 /* Verbs headers do not support -pedantic. */
20 #pragma GCC diagnostic ignored "-Wpedantic"
22 #include <infiniband/mlx5dv.h>
23 #include <infiniband/verbs.h>
25 #pragma GCC diagnostic error "-Wpedantic"
28 #include <rte_config.h>
30 #include "mlx5_autoconf.h"
31 #include "mlx5_glue.h"
34 mlx5_glue_fork_init(void)
36 return ibv_fork_init();
39 static struct ibv_pd *
40 mlx5_glue_alloc_pd(struct ibv_context *context)
42 return ibv_alloc_pd(context);
46 mlx5_glue_dealloc_pd(struct ibv_pd *pd)
48 return ibv_dealloc_pd(pd);
51 static struct ibv_device **
52 mlx5_glue_get_device_list(int *num_devices)
54 return ibv_get_device_list(num_devices);
58 mlx5_glue_free_device_list(struct ibv_device **list)
60 ibv_free_device_list(list);
63 static struct ibv_context *
64 mlx5_glue_open_device(struct ibv_device *device)
66 return ibv_open_device(device);
70 mlx5_glue_close_device(struct ibv_context *context)
72 return ibv_close_device(context);
76 mlx5_glue_query_device(struct ibv_context *context,
77 struct ibv_device_attr *device_attr)
79 return ibv_query_device(context, device_attr);
83 mlx5_glue_query_device_ex(struct ibv_context *context,
84 const struct ibv_query_device_ex_input *input,
85 struct ibv_device_attr_ex *attr)
87 return ibv_query_device_ex(context, input, attr);
91 mlx5_glue_query_port(struct ibv_context *context, uint8_t port_num,
92 struct ibv_port_attr *port_attr)
94 return ibv_query_port(context, port_num, port_attr);
97 static struct ibv_comp_channel *
98 mlx5_glue_create_comp_channel(struct ibv_context *context)
100 return ibv_create_comp_channel(context);
104 mlx5_glue_destroy_comp_channel(struct ibv_comp_channel *channel)
106 return ibv_destroy_comp_channel(channel);
109 static struct ibv_cq *
110 mlx5_glue_create_cq(struct ibv_context *context, int cqe, void *cq_context,
111 struct ibv_comp_channel *channel, int comp_vector)
113 return ibv_create_cq(context, cqe, cq_context, channel, comp_vector);
117 mlx5_glue_destroy_cq(struct ibv_cq *cq)
119 return ibv_destroy_cq(cq);
123 mlx5_glue_get_cq_event(struct ibv_comp_channel *channel, struct ibv_cq **cq,
126 return ibv_get_cq_event(channel, cq, cq_context);
130 mlx5_glue_ack_cq_events(struct ibv_cq *cq, unsigned int nevents)
132 ibv_ack_cq_events(cq, nevents);
135 static struct ibv_rwq_ind_table *
136 mlx5_glue_create_rwq_ind_table(struct ibv_context *context,
137 struct ibv_rwq_ind_table_init_attr *init_attr)
139 return ibv_create_rwq_ind_table(context, init_attr);
143 mlx5_glue_destroy_rwq_ind_table(struct ibv_rwq_ind_table *rwq_ind_table)
145 return ibv_destroy_rwq_ind_table(rwq_ind_table);
148 static struct ibv_wq *
149 mlx5_glue_create_wq(struct ibv_context *context,
150 struct ibv_wq_init_attr *wq_init_attr)
152 return ibv_create_wq(context, wq_init_attr);
156 mlx5_glue_destroy_wq(struct ibv_wq *wq)
158 return ibv_destroy_wq(wq);
161 mlx5_glue_modify_wq(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr)
163 return ibv_modify_wq(wq, wq_attr);
166 static struct ibv_flow *
167 mlx5_glue_create_flow(struct ibv_qp *qp, struct ibv_flow_attr *flow)
169 return ibv_create_flow(qp, flow);
173 mlx5_glue_destroy_flow(struct ibv_flow *flow_id)
175 return ibv_destroy_flow(flow_id);
179 mlx5_glue_destroy_flow_action(void *action)
181 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
182 #ifdef HAVE_MLX5DV_DR
183 return mlx5dv_dr_destroy_action(action);
185 struct mlx5dv_flow_action_attr *attr = action;
187 switch (attr->type) {
188 case MLX5DV_FLOW_ACTION_TAG:
191 res = ibv_destroy_flow_action(attr->action);
203 static struct ibv_qp *
204 mlx5_glue_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *qp_init_attr)
206 return ibv_create_qp(pd, qp_init_attr);
209 static struct ibv_qp *
210 mlx5_glue_create_qp_ex(struct ibv_context *context,
211 struct ibv_qp_init_attr_ex *qp_init_attr_ex)
213 return ibv_create_qp_ex(context, qp_init_attr_ex);
217 mlx5_glue_destroy_qp(struct ibv_qp *qp)
219 return ibv_destroy_qp(qp);
223 mlx5_glue_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
225 return ibv_modify_qp(qp, attr, attr_mask);
228 static struct ibv_mr *
229 mlx5_glue_reg_mr(struct ibv_pd *pd, void *addr, size_t length, int access)
231 return ibv_reg_mr(pd, addr, length, access);
235 mlx5_glue_dereg_mr(struct ibv_mr *mr)
237 return ibv_dereg_mr(mr);
240 static struct ibv_counter_set *
241 mlx5_glue_create_counter_set(struct ibv_context *context,
242 struct ibv_counter_set_init_attr *init_attr)
244 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
249 return ibv_create_counter_set(context, init_attr);
254 mlx5_glue_destroy_counter_set(struct ibv_counter_set *cs)
256 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
260 return ibv_destroy_counter_set(cs);
265 mlx5_glue_describe_counter_set(struct ibv_context *context,
266 uint16_t counter_set_id,
267 struct ibv_counter_set_description *cs_desc)
269 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
271 (void)counter_set_id;
275 return ibv_describe_counter_set(context, counter_set_id, cs_desc);
280 mlx5_glue_query_counter_set(struct ibv_query_counter_set_attr *query_attr,
281 struct ibv_counter_set_data *cs_data)
283 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V42
288 return ibv_query_counter_set(query_attr, cs_data);
292 static struct ibv_counters *
293 mlx5_glue_create_counters(struct ibv_context *context,
294 struct ibv_counters_init_attr *init_attr)
296 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
301 return ibv_create_counters(context, init_attr);
306 mlx5_glue_destroy_counters(struct ibv_counters *counters)
308 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
312 return ibv_destroy_counters(counters);
317 mlx5_glue_attach_counters(struct ibv_counters *counters,
318 struct ibv_counter_attach_attr *attr,
319 struct ibv_flow *flow)
321 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
327 return ibv_attach_counters_point_flow(counters, attr, flow);
332 mlx5_glue_query_counters(struct ibv_counters *counters,
333 uint64_t *counters_value,
337 #ifndef HAVE_IBV_DEVICE_COUNTERS_SET_V45
339 (void)counters_value;
344 return ibv_read_counters(counters, counters_value, ncounters, flags);
349 mlx5_glue_ack_async_event(struct ibv_async_event *event)
351 ibv_ack_async_event(event);
355 mlx5_glue_get_async_event(struct ibv_context *context,
356 struct ibv_async_event *event)
358 return ibv_get_async_event(context, event);
362 mlx5_glue_port_state_str(enum ibv_port_state port_state)
364 return ibv_port_state_str(port_state);
367 static struct ibv_cq *
368 mlx5_glue_cq_ex_to_cq(struct ibv_cq_ex *cq)
370 return ibv_cq_ex_to_cq(cq);
374 mlx5_glue_dr_create_flow_action_dest_flow_tbl(void *tbl)
376 #ifdef HAVE_MLX5DV_DR
377 return mlx5dv_dr_create_action_dest_flow_table(tbl);
385 mlx5_glue_dr_create_flow_action_dest_vport(void *ns, uint32_t vport)
387 #ifdef HAVE_MLX5DV_DR_ESWITCH
388 return mlx5dv_dr_create_action_dest_vport(ns, vport);
397 mlx5_glue_dr_create_flow_tbl(void *ns, uint32_t level)
399 #ifdef HAVE_MLX5DV_DR
400 return mlx5dv_dr_create_ft(ns, level);
409 mlx5_glue_dr_destroy_flow_tbl(void *tbl)
411 #ifdef HAVE_MLX5DV_DR
412 return mlx5dv_dr_destroy_ft(tbl);
420 mlx5_glue_dr_create_ns(struct ibv_context *ctx,
421 enum mlx5dv_dr_ns_domain domain)
423 #ifdef HAVE_MLX5DV_DR
424 return mlx5dv_dr_create_ns(ctx, domain);
433 mlx5_glue_dr_destroy_ns(void *ns)
435 #ifdef HAVE_MLX5DV_DR
436 return mlx5dv_dr_destroy_ns(ns);
443 static struct ibv_cq_ex *
444 mlx5_glue_dv_create_cq(struct ibv_context *context,
445 struct ibv_cq_init_attr_ex *cq_attr,
446 struct mlx5dv_cq_init_attr *mlx5_cq_attr)
448 return mlx5dv_create_cq(context, cq_attr, mlx5_cq_attr);
451 static struct ibv_wq *
452 mlx5_glue_dv_create_wq(struct ibv_context *context,
453 struct ibv_wq_init_attr *wq_attr,
454 struct mlx5dv_wq_init_attr *mlx5_wq_attr)
456 #ifndef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT
462 return mlx5dv_create_wq(context, wq_attr, mlx5_wq_attr);
467 mlx5_glue_dv_query_device(struct ibv_context *ctx,
468 struct mlx5dv_context *attrs_out)
470 return mlx5dv_query_device(ctx, attrs_out);
474 mlx5_glue_dv_set_context_attr(struct ibv_context *ibv_ctx,
475 enum mlx5dv_set_ctx_attr_type type, void *attr)
477 return mlx5dv_set_context_attr(ibv_ctx, type, attr);
481 mlx5_glue_dv_init_obj(struct mlx5dv_obj *obj, uint64_t obj_type)
483 return mlx5dv_init_obj(obj, obj_type);
486 static struct ibv_qp *
487 mlx5_glue_dv_create_qp(struct ibv_context *context,
488 struct ibv_qp_init_attr_ex *qp_init_attr_ex,
489 struct mlx5dv_qp_init_attr *dv_qp_init_attr)
491 #ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
492 return mlx5dv_create_qp(context, qp_init_attr_ex, dv_qp_init_attr);
495 (void)qp_init_attr_ex;
496 (void)dv_qp_init_attr;
502 mlx5_glue_dv_create_flow_matcher(struct ibv_context *context,
503 struct mlx5dv_flow_matcher_attr *matcher_attr,
506 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
507 #ifdef HAVE_MLX5DV_DR
509 return mlx5dv_dr_create_matcher(tbl, matcher_attr->priority,
510 matcher_attr->match_criteria_enable,
511 matcher_attr->match_mask);
514 return mlx5dv_create_flow_matcher(context, matcher_attr);
525 mlx5_glue_dv_create_flow(void *matcher,
530 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
531 #ifdef HAVE_MLX5DV_DR
532 return mlx5dv_dr_create_rule(matcher, match_value, num_actions,
533 (struct mlx5dv_dr_action **)actions);
535 struct mlx5dv_flow_action_attr actions_attr[8];
539 for (size_t i = 0; i < num_actions; i++)
541 *((struct mlx5dv_flow_action_attr *)(actions[i]));
542 return mlx5dv_create_flow(matcher, match_value,
543 num_actions, actions_attr);
555 mlx5_glue_dv_create_flow_action_counter(void *counter_obj, uint32_t offset)
557 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
558 #ifdef HAVE_MLX5DV_DR
559 return mlx5dv_dr_create_action_devx_counter(counter_obj, offset);
561 struct mlx5dv_flow_action_attr *action;
564 action = malloc(sizeof(*action));
567 action->type = MLX5DV_FLOW_ACTION_COUNTERS_DEVX;
568 action->obj = counter_obj;
579 mlx5_glue_dv_create_flow_action_dest_ibv_qp(void *qp)
581 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
582 #ifdef HAVE_MLX5DV_DR
583 return mlx5dv_dr_create_action_dest_ibv_qp(qp);
585 struct mlx5dv_flow_action_attr *action;
587 action = malloc(sizeof(*action));
590 action->type = MLX5DV_FLOW_ACTION_DEST_IBV_QP;
601 mlx5_glue_dv_create_flow_action_modify_header
602 (struct ibv_context *ctx,
603 enum mlx5dv_flow_table_type ft_type,
604 void *ns, uint64_t flags,
608 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
609 #ifdef HAVE_MLX5DV_DR
612 return mlx5dv_dr_create_action_modify_header(ns, flags, actions_sz,
615 struct mlx5dv_flow_action_attr *action;
619 action = malloc(sizeof(*action));
622 action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
623 action->action = mlx5dv_create_flow_action_modify_header
624 (ctx, actions_sz, actions, ft_type);
639 mlx5_glue_dv_create_flow_action_packet_reformat
640 (struct ibv_context *ctx,
641 enum mlx5dv_flow_action_packet_reformat_type reformat_type,
642 enum mlx5dv_flow_table_type ft_type, struct mlx5dv_dr_ns *ns,
643 uint32_t flags, size_t data_sz, void *data)
645 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
646 #ifdef HAVE_MLX5DV_DR
649 return mlx5dv_dr_create_action_packet_reformat(ns, flags,
650 reformat_type, data_sz,
655 struct mlx5dv_flow_action_attr *action;
657 action = malloc(sizeof(*action));
660 action->type = MLX5DV_FLOW_ACTION_IBV_FLOW_ACTION;
661 action->action = mlx5dv_create_flow_action_packet_reformat
662 (ctx, data_sz, data, reformat_type, ft_type);
678 mlx5_glue_dv_create_flow_action_tag(uint32_t tag)
680 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
681 #ifdef HAVE_MLX5DV_DR
682 return mlx5dv_dr_create_action_tag(tag);
684 struct mlx5dv_flow_action_attr *action;
685 action = malloc(sizeof(*action));
688 action->type = MLX5DV_FLOW_ACTION_TAG;
689 action->tag_value = tag;
698 mlx5_glue_dv_destroy_flow(void *flow_id)
700 #ifdef HAVE_MLX5DV_DR
701 return mlx5dv_dr_destroy_rule(flow_id);
703 return ibv_destroy_flow(flow_id);
708 mlx5_glue_dv_destroy_flow_matcher(void *matcher)
710 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
711 #ifdef HAVE_MLX5DV_DR
712 return mlx5dv_dr_destroy_matcher(matcher);
714 return mlx5dv_destroy_flow_matcher(matcher);
722 static struct ibv_context *
723 mlx5_glue_dv_open_device(struct ibv_device *device)
725 #ifdef HAVE_IBV_DEVX_OBJ
726 return mlx5dv_open_device(device,
727 &(struct mlx5dv_context_attr){
728 .flags = MLX5DV_CONTEXT_FLAGS_DEVX,
736 static struct mlx5dv_devx_obj *
737 mlx5_glue_devx_obj_create(struct ibv_context *ctx,
738 const void *in, size_t inlen,
739 void *out, size_t outlen)
741 #ifdef HAVE_IBV_DEVX_OBJ
742 return mlx5dv_devx_obj_create(ctx, in, inlen, out, outlen);
754 mlx5_glue_devx_obj_destroy(struct mlx5dv_devx_obj *obj)
756 #ifdef HAVE_IBV_DEVX_OBJ
757 return mlx5dv_devx_obj_destroy(obj);
765 mlx5_glue_devx_obj_query(struct mlx5dv_devx_obj *obj,
766 const void *in, size_t inlen,
767 void *out, size_t outlen)
769 #ifdef HAVE_IBV_DEVX_OBJ
770 return mlx5dv_devx_obj_query(obj, in, inlen, out, outlen);
782 mlx5_glue_devx_obj_modify(struct mlx5dv_devx_obj *obj,
783 const void *in, size_t inlen,
784 void *out, size_t outlen)
786 #ifdef HAVE_IBV_DEVX_OBJ
787 return mlx5dv_devx_obj_modify(obj, in, inlen, out, outlen);
799 mlx5_glue_devx_general_cmd(struct ibv_context *ctx,
800 const void *in, size_t inlen,
801 void *out, size_t outlen)
803 #ifdef HAVE_IBV_DEVX_OBJ
804 return mlx5dv_devx_general_cmd(ctx, in, inlen, out, outlen);
815 alignas(RTE_CACHE_LINE_SIZE)
816 const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
817 .version = MLX5_GLUE_VERSION,
818 .fork_init = mlx5_glue_fork_init,
819 .alloc_pd = mlx5_glue_alloc_pd,
820 .dealloc_pd = mlx5_glue_dealloc_pd,
821 .get_device_list = mlx5_glue_get_device_list,
822 .free_device_list = mlx5_glue_free_device_list,
823 .open_device = mlx5_glue_open_device,
824 .close_device = mlx5_glue_close_device,
825 .query_device = mlx5_glue_query_device,
826 .query_device_ex = mlx5_glue_query_device_ex,
827 .query_port = mlx5_glue_query_port,
828 .create_comp_channel = mlx5_glue_create_comp_channel,
829 .destroy_comp_channel = mlx5_glue_destroy_comp_channel,
830 .create_cq = mlx5_glue_create_cq,
831 .destroy_cq = mlx5_glue_destroy_cq,
832 .get_cq_event = mlx5_glue_get_cq_event,
833 .ack_cq_events = mlx5_glue_ack_cq_events,
834 .create_rwq_ind_table = mlx5_glue_create_rwq_ind_table,
835 .destroy_rwq_ind_table = mlx5_glue_destroy_rwq_ind_table,
836 .create_wq = mlx5_glue_create_wq,
837 .destroy_wq = mlx5_glue_destroy_wq,
838 .modify_wq = mlx5_glue_modify_wq,
839 .create_flow = mlx5_glue_create_flow,
840 .destroy_flow = mlx5_glue_destroy_flow,
841 .destroy_flow_action = mlx5_glue_destroy_flow_action,
842 .create_qp = mlx5_glue_create_qp,
843 .create_qp_ex = mlx5_glue_create_qp_ex,
844 .destroy_qp = mlx5_glue_destroy_qp,
845 .modify_qp = mlx5_glue_modify_qp,
846 .reg_mr = mlx5_glue_reg_mr,
847 .dereg_mr = mlx5_glue_dereg_mr,
848 .create_counter_set = mlx5_glue_create_counter_set,
849 .destroy_counter_set = mlx5_glue_destroy_counter_set,
850 .describe_counter_set = mlx5_glue_describe_counter_set,
851 .query_counter_set = mlx5_glue_query_counter_set,
852 .create_counters = mlx5_glue_create_counters,
853 .destroy_counters = mlx5_glue_destroy_counters,
854 .attach_counters = mlx5_glue_attach_counters,
855 .query_counters = mlx5_glue_query_counters,
856 .ack_async_event = mlx5_glue_ack_async_event,
857 .get_async_event = mlx5_glue_get_async_event,
858 .port_state_str = mlx5_glue_port_state_str,
859 .cq_ex_to_cq = mlx5_glue_cq_ex_to_cq,
860 .dr_create_flow_action_dest_flow_tbl =
861 mlx5_glue_dr_create_flow_action_dest_flow_tbl,
862 .dr_create_flow_action_dest_vport =
863 mlx5_glue_dr_create_flow_action_dest_vport,
864 .dr_create_flow_tbl = mlx5_glue_dr_create_flow_tbl,
865 .dr_destroy_flow_tbl = mlx5_glue_dr_destroy_flow_tbl,
866 .dr_create_ns = mlx5_glue_dr_create_ns,
867 .dr_destroy_ns = mlx5_glue_dr_destroy_ns,
868 .dv_create_cq = mlx5_glue_dv_create_cq,
869 .dv_create_wq = mlx5_glue_dv_create_wq,
870 .dv_query_device = mlx5_glue_dv_query_device,
871 .dv_set_context_attr = mlx5_glue_dv_set_context_attr,
872 .dv_init_obj = mlx5_glue_dv_init_obj,
873 .dv_create_qp = mlx5_glue_dv_create_qp,
874 .dv_create_flow_matcher = mlx5_glue_dv_create_flow_matcher,
875 .dv_create_flow = mlx5_glue_dv_create_flow,
876 .dv_create_flow_action_counter =
877 mlx5_glue_dv_create_flow_action_counter,
878 .dv_create_flow_action_dest_ibv_qp =
879 mlx5_glue_dv_create_flow_action_dest_ibv_qp,
880 .dv_create_flow_action_modify_header =
881 mlx5_glue_dv_create_flow_action_modify_header,
882 .dv_create_flow_action_packet_reformat =
883 mlx5_glue_dv_create_flow_action_packet_reformat,
884 .dv_create_flow_action_tag = mlx5_glue_dv_create_flow_action_tag,
885 .dv_destroy_flow = mlx5_glue_dv_destroy_flow,
886 .dv_destroy_flow_matcher = mlx5_glue_dv_destroy_flow_matcher,
887 .dv_open_device = mlx5_glue_dv_open_device,
888 .devx_obj_create = mlx5_glue_devx_obj_create,
889 .devx_obj_destroy = mlx5_glue_devx_obj_destroy,
890 .devx_obj_query = mlx5_glue_devx_obj_query,
891 .devx_obj_modify = mlx5_glue_devx_obj_modify,
892 .devx_general_cmd = mlx5_glue_devx_general_cmd,