drivers/common/mlx5/mlx5_devx_cmds.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /* Copyright 2018 Mellanox Technologies, Ltd */
   3
   4 #include <unistd.h>
   5
   6 #include <rte_errno.h>
   7 #include <rte_malloc.h>
   8
   9 #include "mlx5_prm.h"
  10 #include "mlx5_devx_cmds.h"
  11 #include "mlx5_common_utils.h"
  12
  13
  14 /**
  15  * Perform read access to the registers. Reads data from register
  16  * and writes ones to the specified buffer.
  17  *
  18  * @param[in] ctx
  19  *   Context returned from mlx5 open_device() glue function.
  20  * @param[in] reg_id
  21  *   Register identifier according to the PRM.
  22  * @param[in] arg
  23  *   Register access auxiliary parameter according to the PRM.
  24  * @param[out] data
  25  *   Pointer to the buffer to store read data.
  26  * @param[in] dw_cnt
  27  *   Buffer size in double words.
  28  *
  29  * @return
  30  *   0 on success, a negative value otherwise.
  31  */
  32 int
  33 mlx5_devx_cmd_register_read(void *ctx, uint16_t reg_id, uint32_t arg,
  34                             uint32_t *data, uint32_t dw_cnt)
  35 {
  36         uint32_t in[MLX5_ST_SZ_DW(access_register_in)]   = {0};
  37         uint32_t out[MLX5_ST_SZ_DW(access_register_out) +
  38                      MLX5_ACCESS_REGISTER_DATA_DWORD_MAX] = {0};
  39         int status, rc;
  40
  41         MLX5_ASSERT(data && dw_cnt);
  42         MLX5_ASSERT(dw_cnt <= MLX5_ACCESS_REGISTER_DATA_DWORD_MAX);
  43         if (dw_cnt  > MLX5_ACCESS_REGISTER_DATA_DWORD_MAX) {
  44                 DRV_LOG(ERR, "Not enough  buffer for register read data");
  45                 return -1;
  46         }
  47         MLX5_SET(access_register_in, in, opcode,
  48                  MLX5_CMD_OP_ACCESS_REGISTER_USER);
  49         MLX5_SET(access_register_in, in, op_mod,
  50                                         MLX5_ACCESS_REGISTER_IN_OP_MOD_READ);
  51         MLX5_SET(access_register_in, in, register_id, reg_id);
  52         MLX5_SET(access_register_in, in, argument, arg);
  53         rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in), out,
  54                                          MLX5_ST_SZ_DW(access_register_out) *
  55                                          sizeof(uint32_t) + dw_cnt);
  56         if (rc)
  57                 goto error;
  58         status = MLX5_GET(access_register_out, out, status);
  59         if (status) {
  60                 int syndrome = MLX5_GET(access_register_out, out, syndrome);
  61
  62                 DRV_LOG(DEBUG, "Failed to access NIC register 0x%X, "
  63                                "status %x, syndrome = %x",
  64                                reg_id, status, syndrome);
  65                 return -1;
  66         }
  67         memcpy(data, &out[MLX5_ST_SZ_DW(access_register_out)],
  68                dw_cnt * sizeof(uint32_t));
  69         return 0;
  70 error:
  71         rc = (rc > 0) ? -rc : rc;
  72         return rc;
  73 }
  74
  75 /**
  76  * Allocate flow counters via devx interface.
  77  *
  78  * @param[in] ctx
  79  *   Context returned from mlx5 open_device() glue function.
  80  * @param dcs
  81  *   Pointer to counters properties structure to be filled by the routine.
  82  * @param bulk_n_128
  83  *   Bulk counter numbers in 128 counters units.
  84  *
  85  * @return
  86  *   Pointer to counter object on success, a negative value otherwise and
  87  *   rte_errno is set.
  88  */
  89 struct mlx5_devx_obj *
  90 mlx5_devx_cmd_flow_counter_alloc(void *ctx, uint32_t bulk_n_128)
  91 {
  92         struct mlx5_devx_obj *dcs = rte_zmalloc("dcs", sizeof(*dcs), 0);
  93         uint32_t in[MLX5_ST_SZ_DW(alloc_flow_counter_in)]   = {0};
  94         uint32_t out[MLX5_ST_SZ_DW(alloc_flow_counter_out)] = {0};
  95
  96         if (!dcs) {
  97                 rte_errno = ENOMEM;
  98                 return NULL;
  99         }
 100         MLX5_SET(alloc_flow_counter_in, in, opcode,
 101                  MLX5_CMD_OP_ALLOC_FLOW_COUNTER);
 102         MLX5_SET(alloc_flow_counter_in, in, flow_counter_bulk, bulk_n_128);
 103         dcs->obj = mlx5_glue->devx_obj_create(ctx, in,
 104                                               sizeof(in), out, sizeof(out));
 105         if (!dcs->obj) {
 106                 DRV_LOG(ERR, "Can't allocate counters - error %d", errno);
 107                 rte_errno = errno;
 108                 rte_free(dcs);
 109                 return NULL;
 110         }
 111         dcs->id = MLX5_GET(alloc_flow_counter_out, out, flow_counter_id);
 112         return dcs;
 113 }
 114
 115 /**
 116  * Query flow counters values.
 117  *
 118  * @param[in] dcs
 119  *   devx object that was obtained from mlx5_devx_cmd_fc_alloc.
 120  * @param[in] clear
 121  *   Whether hardware should clear the counters after the query or not.
 122  * @param[in] n_counters
 123  *   0 in case of 1 counter to read, otherwise the counter number to read.
 124  *  @param pkts
 125  *   The number of packets that matched the flow.
 126  *  @param bytes
 127  *    The number of bytes that matched the flow.
 128  *  @param mkey
 129  *   The mkey key for batch query.
 130  *  @param addr
 131  *    The address in the mkey range for batch query.
 132  *  @param cmd_comp
 133  *   The completion object for asynchronous batch query.
 134  *  @param async_id
 135  *    The ID to be returned in the asynchronous batch query response.
 136  *
 137  * @return
 138  *   0 on success, a negative value otherwise.
 139  */
 140 int
 141 mlx5_devx_cmd_flow_counter_query(struct mlx5_devx_obj *dcs,
 142                                  int clear, uint32_t n_counters,
 143                                  uint64_t *pkts, uint64_t *bytes,
 144                                  uint32_t mkey, void *addr,
 145                                  void *cmd_comp,
 146                                  uint64_t async_id)
 147 {
 148         int out_len = MLX5_ST_SZ_BYTES(query_flow_counter_out) +
 149                         MLX5_ST_SZ_BYTES(traffic_counter);
 150         uint32_t out[out_len];
 151         uint32_t in[MLX5_ST_SZ_DW(query_flow_counter_in)] = {0};
 152         void *stats;
 153         int rc;
 154
 155         MLX5_SET(query_flow_counter_in, in, opcode,
 156                  MLX5_CMD_OP_QUERY_FLOW_COUNTER);
 157         MLX5_SET(query_flow_counter_in, in, op_mod, 0);
 158         MLX5_SET(query_flow_counter_in, in, flow_counter_id, dcs->id);
 159         MLX5_SET(query_flow_counter_in, in, clear, !!clear);
 160
 161         if (n_counters) {
 162                 MLX5_SET(query_flow_counter_in, in, num_of_counters,
 163                          n_counters);
 164                 MLX5_SET(query_flow_counter_in, in, dump_to_memory, 1);
 165                 MLX5_SET(query_flow_counter_in, in, mkey, mkey);
 166                 MLX5_SET64(query_flow_counter_in, in, address,
 167                            (uint64_t)(uintptr_t)addr);
 168         }
 169         if (!cmd_comp)
 170                 rc = mlx5_glue->devx_obj_query(dcs->obj, in, sizeof(in), out,
 171                                                out_len);
 172         else
 173                 rc = mlx5_glue->devx_obj_query_async(dcs->obj, in, sizeof(in),
 174                                                      out_len, async_id,
 175                                                      cmd_comp);
 176         if (rc) {
 177                 DRV_LOG(ERR, "Failed to query devx counters with rc %d", rc);
 178                 rte_errno = rc;
 179                 return -rc;
 180         }
 181         if (!n_counters) {
 182                 stats = MLX5_ADDR_OF(query_flow_counter_out,
 183                                      out, flow_statistics);
 184                 *pkts = MLX5_GET64(traffic_counter, stats, packets);
 185                 *bytes = MLX5_GET64(traffic_counter, stats, octets);
 186         }
 187         return 0;
 188 }
 189
 190 /**
 191  * Create a new mkey.
 192  *
 193  * @param[in] ctx
 194  *   Context returned from mlx5 open_device() glue function.
 195  * @param[in] attr
 196  *   Attributes of the requested mkey.
 197  *
 198  * @return
 199  *   Pointer to Devx mkey on success, a negative value otherwise and rte_errno
 200  *   is set.
 201  */
 202 struct mlx5_devx_obj *
 203 mlx5_devx_cmd_mkey_create(void *ctx,
 204                           struct mlx5_devx_mkey_attr *attr)
 205 {
 206         struct mlx5_klm *klm_array = attr->klm_array;
 207         int klm_num = attr->klm_num;
 208         int in_size_dw = MLX5_ST_SZ_DW(create_mkey_in) +
 209                      (klm_num ? RTE_ALIGN(klm_num, 4) : 0) * MLX5_ST_SZ_DW(klm);
 210         uint32_t in[in_size_dw];
 211         uint32_t out[MLX5_ST_SZ_DW(create_mkey_out)] = {0};
 212         void *mkc;
 213         struct mlx5_devx_obj *mkey = rte_zmalloc("mkey", sizeof(*mkey), 0);
 214         size_t pgsize;
 215         uint32_t translation_size;
 216
 217         if (!mkey) {
 218                 rte_errno = ENOMEM;
 219                 return NULL;
 220         }
 221         memset(in, 0, in_size_dw * 4);
 222         pgsize = sysconf(_SC_PAGESIZE);
 223         MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
 224         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
 225         if (klm_num > 0) {
 226                 int i;
 227                 uint8_t *klm = (uint8_t *)MLX5_ADDR_OF(create_mkey_in, in,
 228                                                        klm_pas_mtt);
 229                 translation_size = RTE_ALIGN(klm_num, 4);
 230                 for (i = 0; i < klm_num; i++) {
 231                         MLX5_SET(klm, klm, byte_count, klm_array[i].byte_count);
 232                         MLX5_SET(klm, klm, mkey, klm_array[i].mkey);
 233                         MLX5_SET64(klm, klm, address, klm_array[i].address);
 234                         klm += MLX5_ST_SZ_BYTES(klm);
 235                 }
 236                 for (; i < (int)translation_size; i++) {
 237                         MLX5_SET(klm, klm, mkey, 0x0);
 238                         MLX5_SET64(klm, klm, address, 0x0);
 239                         klm += MLX5_ST_SZ_BYTES(klm);
 240                 }
 241                 MLX5_SET(mkc, mkc, access_mode_1_0, attr->log_entity_size ?
 242                          MLX5_MKC_ACCESS_MODE_KLM_FBS :
 243                          MLX5_MKC_ACCESS_MODE_KLM);
 244                 MLX5_SET(mkc, mkc, log_page_size, attr->log_entity_size);
 245         } else {
 246                 translation_size = (RTE_ALIGN(attr->size, pgsize) * 8) / 16;
 247                 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
 248                 MLX5_SET(mkc, mkc, log_page_size, rte_log2_u32(pgsize));
 249         }
 250         MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
 251                  translation_size);
 252         MLX5_SET(create_mkey_in, in, mkey_umem_id, attr->umem_id);
 253         MLX5_SET(create_mkey_in, in, pg_access, attr->pg_access);
 254         MLX5_SET(mkc, mkc, lw, 0x1);
 255         MLX5_SET(mkc, mkc, lr, 0x1);
 256         MLX5_SET(mkc, mkc, qpn, 0xffffff);
 257         MLX5_SET(mkc, mkc, pd, attr->pd);
 258         MLX5_SET(mkc, mkc, mkey_7_0, attr->umem_id & 0xFF);
 259         MLX5_SET(mkc, mkc, translations_octword_size, translation_size);
 260         if (attr->relaxed_ordering == 1) {
 261                 MLX5_SET(mkc, mkc, relaxed_ordering_write, 0x1);
 262                 MLX5_SET(mkc, mkc, relaxed_ordering_read, 0x1);
 263         }
 264         MLX5_SET64(mkc, mkc, start_addr, attr->addr);
 265         MLX5_SET64(mkc, mkc, len, attr->size);
 266         mkey->obj = mlx5_glue->devx_obj_create(ctx, in, in_size_dw * 4, out,
 267                                                sizeof(out));
 268         if (!mkey->obj) {
 269                 DRV_LOG(ERR, "Can't create %sdirect mkey - error %d\n",
 270                         klm_num ? "an in" : "a ", errno);
 271                 rte_errno = errno;
 272                 rte_free(mkey);
 273                 return NULL;
 274         }
 275         mkey->id = MLX5_GET(create_mkey_out, out, mkey_index);
 276         mkey->id = (mkey->id << 8) | (attr->umem_id & 0xFF);
 277         return mkey;
 278 }
 279
 280 /**
 281  * Get status of devx command response.
 282  * Mainly used for asynchronous commands.
 283  *
 284  * @param[in] out
 285  *   The out response buffer.
 286  *
 287  * @return
 288  *   0 on success, non-zero value otherwise.
 289  */
 290 int
 291 mlx5_devx_get_out_command_status(void *out)
 292 {
 293         int status;
 294
 295         if (!out)
 296                 return -EINVAL;
 297         status = MLX5_GET(query_flow_counter_out, out, status);
 298         if (status) {
 299                 int syndrome = MLX5_GET(query_flow_counter_out, out, syndrome);
 300
 301                 DRV_LOG(ERR, "Bad devX status %x, syndrome = %x", status,
 302                         syndrome);
 303         }
 304         return status;
 305 }
 306
 307 /**
 308  * Destroy any object allocated by a Devx API.
 309  *
 310  * @param[in] obj
 311  *   Pointer to a general object.
 312  *
 313  * @return
 314  *   0 on success, a negative value otherwise.
 315  */
 316 int
 317 mlx5_devx_cmd_destroy(struct mlx5_devx_obj *obj)
 318 {
 319         int ret;
 320
 321         if (!obj)
 322                 return 0;
 323         ret =  mlx5_glue->devx_obj_destroy(obj->obj);
 324         rte_free(obj);
 325         return ret;
 326 }
 327
 328 /**
 329  * Query NIC vport context.
 330  * Fills minimal inline attribute.
 331  *
 332  * @param[in] ctx
 333  *   ibv contexts returned from mlx5dv_open_device.
 334  * @param[in] vport
 335  *   vport index
 336  * @param[out] attr
 337  *   Attributes device values.
 338  *
 339  * @return
 340  *   0 on success, a negative value otherwise.
 341  */
 342 static int
 343 mlx5_devx_cmd_query_nic_vport_context(void *ctx,
 344                                       unsigned int vport,
 345                                       struct mlx5_hca_attr *attr)
 346 {
 347         uint32_t in[MLX5_ST_SZ_DW(query_nic_vport_context_in)] = {0};
 348         uint32_t out[MLX5_ST_SZ_DW(query_nic_vport_context_out)] = {0};
 349         void *vctx;
 350         int status, syndrome, rc;
 351
 352         /* Query NIC vport context to determine inline mode. */
 353         MLX5_SET(query_nic_vport_context_in, in, opcode,
 354                  MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT);
 355         MLX5_SET(query_nic_vport_context_in, in, vport_number, vport);
 356         if (vport)
 357                 MLX5_SET(query_nic_vport_context_in, in, other_vport, 1);
 358         rc = mlx5_glue->devx_general_cmd(ctx,
 359                                          in, sizeof(in),
 360                                          out, sizeof(out));
 361         if (rc)
 362                 goto error;
 363         status = MLX5_GET(query_nic_vport_context_out, out, status);
 364         syndrome = MLX5_GET(query_nic_vport_context_out, out, syndrome);
 365         if (status) {
 366                 DRV_LOG(DEBUG, "Failed to query NIC vport context, "
 367                         "status %x, syndrome = %x",
 368                         status, syndrome);
 369                 return -1;
 370         }
 371         vctx = MLX5_ADDR_OF(query_nic_vport_context_out, out,
 372                             nic_vport_context);
 373         attr->vport_inline_mode = MLX5_GET(nic_vport_context, vctx,
 374                                            min_wqe_inline_mode);
 375         return 0;
 376 error:
 377         rc = (rc > 0) ? -rc : rc;
 378         return rc;
 379 }
 380
 381 /**
 382  * Query NIC vDPA attributes.
 383  *
 384  * @param[in] ctx
 385  *   Context returned from mlx5 open_device() glue function.
 386  * @param[out] vdpa_attr
 387  *   vDPA Attributes structure to fill.
 388  */
 389 static void
 390 mlx5_devx_cmd_query_hca_vdpa_attr(void *ctx,
 391                                   struct mlx5_hca_vdpa_attr *vdpa_attr)
 392 {
 393         uint32_t in[MLX5_ST_SZ_DW(query_hca_cap_in)] = {0};
 394         uint32_t out[MLX5_ST_SZ_DW(query_hca_cap_out)] = {0};
 395         void *hcattr = MLX5_ADDR_OF(query_hca_cap_out, out, capability);
 396         int status, syndrome, rc;
 397
 398         MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
 399         MLX5_SET(query_hca_cap_in, in, op_mod,
 400                  MLX5_GET_HCA_CAP_OP_MOD_VDPA_EMULATION |
 401                  MLX5_HCA_CAP_OPMOD_GET_CUR);
 402         rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in), out, sizeof(out));
 403         status = MLX5_GET(query_hca_cap_out, out, status);
 404         syndrome = MLX5_GET(query_hca_cap_out, out, syndrome);
 405         if (rc || status) {
 406                 RTE_LOG(DEBUG, PMD, "Failed to query devx VDPA capabilities,"
 407                         " status %x, syndrome = %x", status, syndrome);
 408                 vdpa_attr->valid = 0;
 409         } else {
 410                 vdpa_attr->valid = 1;
 411                 vdpa_attr->desc_tunnel_offload_type =
 412                         MLX5_GET(virtio_emulation_cap, hcattr,
 413                                  desc_tunnel_offload_type);
 414                 vdpa_attr->eth_frame_offload_type =
 415                         MLX5_GET(virtio_emulation_cap, hcattr,
 416                                  eth_frame_offload_type);
 417                 vdpa_attr->virtio_version_1_0 =
 418                         MLX5_GET(virtio_emulation_cap, hcattr,
 419                                  virtio_version_1_0);
 420                 vdpa_attr->tso_ipv4 = MLX5_GET(virtio_emulation_cap, hcattr,
 421                                                tso_ipv4);
 422                 vdpa_attr->tso_ipv6 = MLX5_GET(virtio_emulation_cap, hcattr,
 423                                                tso_ipv6);
 424                 vdpa_attr->tx_csum = MLX5_GET(virtio_emulation_cap, hcattr,
 425                                               tx_csum);
 426                 vdpa_attr->rx_csum = MLX5_GET(virtio_emulation_cap, hcattr,
 427                                               rx_csum);
 428                 vdpa_attr->event_mode = MLX5_GET(virtio_emulation_cap, hcattr,
 429                                                  event_mode);
 430                 vdpa_attr->virtio_queue_type =
 431                         MLX5_GET(virtio_emulation_cap, hcattr,
 432                                  virtio_queue_type);
 433                 vdpa_attr->log_doorbell_stride =
 434                         MLX5_GET(virtio_emulation_cap, hcattr,
 435                                  log_doorbell_stride);
 436                 vdpa_attr->log_doorbell_bar_size =
 437                         MLX5_GET(virtio_emulation_cap, hcattr,
 438                                  log_doorbell_bar_size);
 439                 vdpa_attr->doorbell_bar_offset =
 440                         MLX5_GET64(virtio_emulation_cap, hcattr,
 441                                    doorbell_bar_offset);
 442                 vdpa_attr->max_num_virtio_queues =
 443                         MLX5_GET(virtio_emulation_cap, hcattr,
 444                                  max_num_virtio_queues);
 445                 vdpa_attr->umems[0].a = MLX5_GET(virtio_emulation_cap, hcattr,
 446                                                  umem_1_buffer_param_a);
 447                 vdpa_attr->umems[0].b = MLX5_GET(virtio_emulation_cap, hcattr,
 448                                                  umem_1_buffer_param_b);
 449                 vdpa_attr->umems[1].a = MLX5_GET(virtio_emulation_cap, hcattr,
 450                                                  umem_2_buffer_param_a);
 451                 vdpa_attr->umems[1].b = MLX5_GET(virtio_emulation_cap, hcattr,
 452                                                  umem_2_buffer_param_b);
 453                 vdpa_attr->umems[2].a = MLX5_GET(virtio_emulation_cap, hcattr,
 454                                                  umem_3_buffer_param_a);
 455                 vdpa_attr->umems[2].b = MLX5_GET(virtio_emulation_cap, hcattr,
 456                                                  umem_3_buffer_param_b);
 457         }
 458 }
 459
 460 /**
 461  * Query HCA attributes.
 462  * Using those attributes we can check on run time if the device
 463  * is having the required capabilities.
 464  *
 465  * @param[in] ctx
 466  *   Context returned from mlx5 open_device() glue function.
 467  * @param[out] attr
 468  *   Attributes device values.
 469  *
 470  * @return
 471  *   0 on success, a negative value otherwise.
 472  */
 473 int
 474 mlx5_devx_cmd_query_hca_attr(void *ctx,
 475                              struct mlx5_hca_attr *attr)
 476 {
 477         uint32_t in[MLX5_ST_SZ_DW(query_hca_cap_in)] = {0};
 478         uint32_t out[MLX5_ST_SZ_DW(query_hca_cap_out)] = {0};
 479         void *hcattr;
 480         int status, syndrome, rc, i;
 481
 482         MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
 483         MLX5_SET(query_hca_cap_in, in, op_mod,
 484                  MLX5_GET_HCA_CAP_OP_MOD_GENERAL_DEVICE |
 485                  MLX5_HCA_CAP_OPMOD_GET_CUR);
 486
 487         rc = mlx5_glue->devx_general_cmd(ctx,
 488                                          in, sizeof(in), out, sizeof(out));
 489         if (rc)
 490                 goto error;
 491         status = MLX5_GET(query_hca_cap_out, out, status);
 492         syndrome = MLX5_GET(query_hca_cap_out, out, syndrome);
 493         if (status) {
 494                 DRV_LOG(DEBUG, "Failed to query devx HCA capabilities, "
 495                         "status %x, syndrome = %x",
 496                         status, syndrome);
 497                 return -1;
 498         }
 499         hcattr = MLX5_ADDR_OF(query_hca_cap_out, out, capability);
 500         attr->flow_counter_bulk_alloc_bitmap =
 501                         MLX5_GET(cmd_hca_cap, hcattr, flow_counter_bulk_alloc);
 502         attr->flow_counters_dump = MLX5_GET(cmd_hca_cap, hcattr,
 503                                             flow_counters_dump);
 504         attr->log_max_rqt_size = MLX5_GET(cmd_hca_cap, hcattr,
 505                                           log_max_rqt_size);
 506         attr->eswitch_manager = MLX5_GET(cmd_hca_cap, hcattr, eswitch_manager);
 507         attr->hairpin = MLX5_GET(cmd_hca_cap, hcattr, hairpin);
 508         attr->log_max_hairpin_queues = MLX5_GET(cmd_hca_cap, hcattr,
 509                                                 log_max_hairpin_queues);
 510         attr->log_max_hairpin_wq_data_sz = MLX5_GET(cmd_hca_cap, hcattr,
 511                                                     log_max_hairpin_wq_data_sz);
 512         attr->log_max_hairpin_num_packets = MLX5_GET
 513                 (cmd_hca_cap, hcattr, log_min_hairpin_wq_data_sz);
 514         attr->vhca_id = MLX5_GET(cmd_hca_cap, hcattr, vhca_id);
 515         attr->relaxed_ordering_write = MLX5_GET(cmd_hca_cap, hcattr,
 516                         relaxed_ordering_write);
 517         attr->relaxed_ordering_read = MLX5_GET(cmd_hca_cap, hcattr,
 518                         relaxed_ordering_read);
 519         attr->eth_net_offloads = MLX5_GET(cmd_hca_cap, hcattr,
 520                                           eth_net_offloads);
 521         attr->eth_virt = MLX5_GET(cmd_hca_cap, hcattr, eth_virt);
 522         attr->flex_parser_protocols = MLX5_GET(cmd_hca_cap, hcattr,
 523                                                flex_parser_protocols);
 524         attr->qos.sup = MLX5_GET(cmd_hca_cap, hcattr, qos);
 525         attr->vdpa.valid = !!(MLX5_GET64(cmd_hca_cap, hcattr,
 526                                          general_obj_types) &
 527                               MLX5_GENERAL_OBJ_TYPES_CAP_VIRTQ_NET_Q);
 528         attr->vdpa.queue_counters_valid = !!(MLX5_GET64(cmd_hca_cap, hcattr,
 529                                                         general_obj_types) &
 530                                   MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_Q_COUNTERS);
 531         attr->wqe_index_ignore = MLX5_GET(cmd_hca_cap, hcattr,
 532                                           wqe_index_ignore_cap);
 533         attr->cross_channel = MLX5_GET(cmd_hca_cap, hcattr, cd);
 534         attr->non_wire_sq = MLX5_GET(cmd_hca_cap, hcattr, non_wire_sq);
 535         attr->log_max_static_sq_wq = MLX5_GET(cmd_hca_cap, hcattr,
 536                                               log_max_static_sq_wq);
 537         attr->dev_freq_khz = MLX5_GET(cmd_hca_cap, hcattr,
 538                                       device_frequency_khz);
 539         attr->regex = MLX5_GET(cmd_hca_cap, hcattr, regexp);
 540         attr->regexp_num_of_engines = MLX5_GET(cmd_hca_cap, hcattr,
 541                                                regexp_num_of_engines);
 542         if (attr->qos.sup) {
 543                 MLX5_SET(query_hca_cap_in, in, op_mod,
 544                          MLX5_GET_HCA_CAP_OP_MOD_QOS_CAP |
 545                          MLX5_HCA_CAP_OPMOD_GET_CUR);
 546                 rc = mlx5_glue->devx_general_cmd(ctx, in, sizeof(in),
 547                                                  out, sizeof(out));
 548                 if (rc)
 549                         goto error;
 550                 if (status) {
 551                         DRV_LOG(DEBUG, "Failed to query devx QOS capabilities,"
 552                                 " status %x, syndrome = %x",
 553                                 status, syndrome);
 554                         return -1;
 555                 }
 556                 hcattr = MLX5_ADDR_OF(query_hca_cap_out, out, capability);
 557                 attr->qos.srtcm_sup =
 558                                 MLX5_GET(qos_cap, hcattr, flow_meter_srtcm);
 559                 attr->qos.log_max_flow_meter =
 560                                 MLX5_GET(qos_cap, hcattr, log_max_flow_meter);
 561                 attr->qos.flow_meter_reg_c_ids =
 562                                 MLX5_GET(qos_cap, hcattr, flow_meter_reg_id);
 563                 attr->qos.flow_meter_reg_share =
 564                                 MLX5_GET(qos_cap, hcattr, flow_meter_reg_share);
 565                 attr->qos.packet_pacing =
 566                                 MLX5_GET(qos_cap, hcattr, packet_pacing);
 567                 attr->qos.wqe_rate_pp =
 568                                 MLX5_GET(qos_cap, hcattr, wqe_rate_pp);
 569         }
 570         if (attr->vdpa.valid)
 571                 mlx5_devx_cmd_query_hca_vdpa_attr(ctx, &attr->vdpa);
 572         if (!attr->eth_net_offloads)
 573                 return 0;
 574
 575         /* Query HCA offloads for Ethernet protocol. */
 576         memset(in, 0, sizeof(in));
 577         memset(out, 0, sizeof(out));
 578         MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
 579         MLX5_SET(query_hca_cap_in, in, op_mod,
 580                  MLX5_GET_HCA_CAP_OP_MOD_ETHERNET_OFFLOAD_CAPS |
 581                  MLX5_HCA_CAP_OPMOD_GET_CUR);
 582
 583         rc = mlx5_glue->devx_general_cmd(ctx,
 584                                          in, sizeof(in),
 585                                          out, sizeof(out));
 586         if (rc) {
 587                 attr->eth_net_offloads = 0;
 588                 goto error;
 589         }
 590         status = MLX5_GET(query_hca_cap_out, out, status);
 591         syndrome = MLX5_GET(query_hca_cap_out, out, syndrome);
 592         if (status) {
 593                 DRV_LOG(DEBUG, "Failed to query devx HCA capabilities, "
 594                         "status %x, syndrome = %x",
 595                         status, syndrome);
 596                 attr->eth_net_offloads = 0;
 597                 return -1;
 598         }
 599         hcattr = MLX5_ADDR_OF(query_hca_cap_out, out, capability);
 600         attr->wqe_vlan_insert = MLX5_GET(per_protocol_networking_offload_caps,
 601                                          hcattr, wqe_vlan_insert);
 602         attr->lro_cap = MLX5_GET(per_protocol_networking_offload_caps, hcattr,
 603                                  lro_cap);
 604         attr->tunnel_lro_gre = MLX5_GET(per_protocol_networking_offload_caps,
 605                                         hcattr, tunnel_lro_gre);
 606         attr->tunnel_lro_vxlan = MLX5_GET(per_protocol_networking_offload_caps,
 607                                           hcattr, tunnel_lro_vxlan);
 608         attr->lro_max_msg_sz_mode = MLX5_GET
 609                                         (per_protocol_networking_offload_caps,
 610                                          hcattr, lro_max_msg_sz_mode);
 611         for (i = 0 ; i < MLX5_LRO_NUM_SUPP_PERIODS ; i++) {
 612                 attr->lro_timer_supported_periods[i] =
 613                         MLX5_GET(per_protocol_networking_offload_caps, hcattr,
 614                                  lro_timer_supported_periods[i]);
 615         }
 616         attr->tunnel_stateless_geneve_rx =
 617                             MLX5_GET(per_protocol_networking_offload_caps,
 618                                      hcattr, tunnel_stateless_geneve_rx);
 619         attr->geneve_max_opt_len =
 620                     MLX5_GET(per_protocol_networking_offload_caps,
 621                              hcattr, max_geneve_opt_len);
 622         attr->wqe_inline_mode = MLX5_GET(per_protocol_networking_offload_caps,
 623                                          hcattr, wqe_inline_mode);
 624         attr->tunnel_stateless_gtp = MLX5_GET
 625                                         (per_protocol_networking_offload_caps,
 626                                          hcattr, tunnel_stateless_gtp);
 627         if (attr->wqe_inline_mode != MLX5_CAP_INLINE_MODE_VPORT_CONTEXT)
 628                 return 0;
 629         if (attr->eth_virt) {
 630                 rc = mlx5_devx_cmd_query_nic_vport_context(ctx, 0, attr);
 631                 if (rc) {
 632                         attr->eth_virt = 0;
 633                         goto error;
 634                 }
 635         }
 636         return 0;
 637 error:
 638         rc = (rc > 0) ? -rc : rc;
 639         return rc;
 640 }
 641
 642 /**
 643  * Query TIS transport domain from QP verbs object using DevX API.
 644  *
 645  * @param[in] qp
 646  *   Pointer to verbs QP returned by ibv_create_qp .
 647  * @param[in] tis_num
 648  *   TIS number of TIS to query.
 649  * @param[out] tis_td
 650  *   Pointer to TIS transport domain variable, to be set by the routine.
 651  *
 652  * @return
 653  *   0 on success, a negative value otherwise.
 654  */
 655 int
 656 mlx5_devx_cmd_qp_query_tis_td(void *qp, uint32_t tis_num,
 657                               uint32_t *tis_td)
 658 {
 659 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 660         uint32_t in[MLX5_ST_SZ_DW(query_tis_in)] = {0};
 661         uint32_t out[MLX5_ST_SZ_DW(query_tis_out)] = {0};
 662         int rc;
 663         void *tis_ctx;
 664
 665         MLX5_SET(query_tis_in, in, opcode, MLX5_CMD_OP_QUERY_TIS);
 666         MLX5_SET(query_tis_in, in, tisn, tis_num);
 667         rc = mlx5_glue->devx_qp_query(qp, in, sizeof(in), out, sizeof(out));
 668         if (rc) {
 669                 DRV_LOG(ERR, "Failed to query QP using DevX");
 670                 return -rc;
 671         };
 672         tis_ctx = MLX5_ADDR_OF(query_tis_out, out, tis_context);
 673         *tis_td = MLX5_GET(tisc, tis_ctx, transport_domain);
 674         return 0;
 675 #else
 676         (void)qp;
 677         (void)tis_num;
 678         (void)tis_td;
 679         return -ENOTSUP;
 680 #endif
 681 }
 682
 683 /**
 684  * Fill WQ data for DevX API command.
 685  * Utility function for use when creating DevX objects containing a WQ.
 686  *
 687  * @param[in] wq_ctx
 688  *   Pointer to WQ context to fill with data.
 689  * @param [in] wq_attr
 690  *   Pointer to WQ attributes structure to fill in WQ context.
 691  */
 692 static void
 693 devx_cmd_fill_wq_data(void *wq_ctx, struct mlx5_devx_wq_attr *wq_attr)
 694 {
 695         MLX5_SET(wq, wq_ctx, wq_type, wq_attr->wq_type);
 696         MLX5_SET(wq, wq_ctx, wq_signature, wq_attr->wq_signature);
 697         MLX5_SET(wq, wq_ctx, end_padding_mode, wq_attr->end_padding_mode);
 698         MLX5_SET(wq, wq_ctx, cd_slave, wq_attr->cd_slave);
 699         MLX5_SET(wq, wq_ctx, hds_skip_first_sge, wq_attr->hds_skip_first_sge);
 700         MLX5_SET(wq, wq_ctx, log2_hds_buf_size, wq_attr->log2_hds_buf_size);
 701         MLX5_SET(wq, wq_ctx, page_offset, wq_attr->page_offset);
 702         MLX5_SET(wq, wq_ctx, lwm, wq_attr->lwm);
 703         MLX5_SET(wq, wq_ctx, pd, wq_attr->pd);
 704         MLX5_SET(wq, wq_ctx, uar_page, wq_attr->uar_page);
 705         MLX5_SET64(wq, wq_ctx, dbr_addr, wq_attr->dbr_addr);
 706         MLX5_SET(wq, wq_ctx, hw_counter, wq_attr->hw_counter);
 707         MLX5_SET(wq, wq_ctx, sw_counter, wq_attr->sw_counter);
 708         MLX5_SET(wq, wq_ctx, log_wq_stride, wq_attr->log_wq_stride);
 709         MLX5_SET(wq, wq_ctx, log_wq_pg_sz, wq_attr->log_wq_pg_sz);
 710         MLX5_SET(wq, wq_ctx, log_wq_sz, wq_attr->log_wq_sz);
 711         MLX5_SET(wq, wq_ctx, dbr_umem_valid, wq_attr->dbr_umem_valid);
 712         MLX5_SET(wq, wq_ctx, wq_umem_valid, wq_attr->wq_umem_valid);
 713         MLX5_SET(wq, wq_ctx, log_hairpin_num_packets,
 714                  wq_attr->log_hairpin_num_packets);
 715         MLX5_SET(wq, wq_ctx, log_hairpin_data_sz, wq_attr->log_hairpin_data_sz);
 716         MLX5_SET(wq, wq_ctx, single_wqe_log_num_of_strides,
 717                  wq_attr->single_wqe_log_num_of_strides);
 718         MLX5_SET(wq, wq_ctx, two_byte_shift_en, wq_attr->two_byte_shift_en);
 719         MLX5_SET(wq, wq_ctx, single_stride_log_num_of_bytes,
 720                  wq_attr->single_stride_log_num_of_bytes);
 721         MLX5_SET(wq, wq_ctx, dbr_umem_id, wq_attr->dbr_umem_id);
 722         MLX5_SET(wq, wq_ctx, wq_umem_id, wq_attr->wq_umem_id);
 723         MLX5_SET64(wq, wq_ctx, wq_umem_offset, wq_attr->wq_umem_offset);
 724 }
 725
 726 /**
 727  * Create RQ using DevX API.
 728  *
 729  * @param[in] ctx
 730  *   Context returned from mlx5 open_device() glue function.
 731  * @param [in] rq_attr
 732  *   Pointer to create RQ attributes structure.
 733  * @param [in] socket
 734  *   CPU socket ID for allocations.
 735  *
 736  * @return
 737  *   The DevX object created, NULL otherwise and rte_errno is set.
 738  */
 739 struct mlx5_devx_obj *
 740 mlx5_devx_cmd_create_rq(void *ctx,
 741                         struct mlx5_devx_create_rq_attr *rq_attr,
 742                         int socket)
 743 {
 744         uint32_t in[MLX5_ST_SZ_DW(create_rq_in)] = {0};
 745         uint32_t out[MLX5_ST_SZ_DW(create_rq_out)] = {0};
 746         void *rq_ctx, *wq_ctx;
 747         struct mlx5_devx_wq_attr *wq_attr;
 748         struct mlx5_devx_obj *rq = NULL;
 749
 750         rq = rte_calloc_socket(__func__, 1, sizeof(*rq), 0, socket);
 751         if (!rq) {
 752                 DRV_LOG(ERR, "Failed to allocate RQ data");
 753                 rte_errno = ENOMEM;
 754                 return NULL;
 755         }
 756         MLX5_SET(create_rq_in, in, opcode, MLX5_CMD_OP_CREATE_RQ);
 757         rq_ctx = MLX5_ADDR_OF(create_rq_in, in, ctx);
 758         MLX5_SET(rqc, rq_ctx, rlky, rq_attr->rlky);
 759         MLX5_SET(rqc, rq_ctx, delay_drop_en, rq_attr->delay_drop_en);
 760         MLX5_SET(rqc, rq_ctx, scatter_fcs, rq_attr->scatter_fcs);
 761         MLX5_SET(rqc, rq_ctx, vsd, rq_attr->vsd);
 762         MLX5_SET(rqc, rq_ctx, mem_rq_type, rq_attr->mem_rq_type);
 763         MLX5_SET(rqc, rq_ctx, state, rq_attr->state);
 764         MLX5_SET(rqc, rq_ctx, flush_in_error_en, rq_attr->flush_in_error_en);
 765         MLX5_SET(rqc, rq_ctx, hairpin, rq_attr->hairpin);
 766         MLX5_SET(rqc, rq_ctx, user_index, rq_attr->user_index);
 767         MLX5_SET(rqc, rq_ctx, cqn, rq_attr->cqn);
 768         MLX5_SET(rqc, rq_ctx, counter_set_id, rq_attr->counter_set_id);
 769         MLX5_SET(rqc, rq_ctx, rmpn, rq_attr->rmpn);
 770         wq_ctx = MLX5_ADDR_OF(rqc, rq_ctx, wq);
 771         wq_attr = &rq_attr->wq_attr;
 772         devx_cmd_fill_wq_data(wq_ctx, wq_attr);
 773         rq->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
 774                                                   out, sizeof(out));
 775         if (!rq->obj) {
 776                 DRV_LOG(ERR, "Failed to create RQ using DevX");
 777                 rte_errno = errno;
 778                 rte_free(rq);
 779                 return NULL;
 780         }
 781         rq->id = MLX5_GET(create_rq_out, out, rqn);
 782         return rq;
 783 }
 784
 785 /**
 786  * Modify RQ using DevX API.
 787  *
 788  * @param[in] rq
 789  *   Pointer to RQ object structure.
 790  * @param [in] rq_attr
 791  *   Pointer to modify RQ attributes structure.
 792  *
 793  * @return
 794  *   0 on success, a negative errno value otherwise and rte_errno is set.
 795  */
 796 int
 797 mlx5_devx_cmd_modify_rq(struct mlx5_devx_obj *rq,
 798                         struct mlx5_devx_modify_rq_attr *rq_attr)
 799 {
 800         uint32_t in[MLX5_ST_SZ_DW(modify_rq_in)] = {0};
 801         uint32_t out[MLX5_ST_SZ_DW(modify_rq_out)] = {0};
 802         void *rq_ctx, *wq_ctx;
 803         int ret;
 804
 805         MLX5_SET(modify_rq_in, in, opcode, MLX5_CMD_OP_MODIFY_RQ);
 806         MLX5_SET(modify_rq_in, in, rq_state, rq_attr->rq_state);
 807         MLX5_SET(modify_rq_in, in, rqn, rq->id);
 808         MLX5_SET64(modify_rq_in, in, modify_bitmask, rq_attr->modify_bitmask);
 809         rq_ctx = MLX5_ADDR_OF(modify_rq_in, in, ctx);
 810         MLX5_SET(rqc, rq_ctx, state, rq_attr->state);
 811         if (rq_attr->modify_bitmask &
 812                         MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS)
 813                 MLX5_SET(rqc, rq_ctx, scatter_fcs, rq_attr->scatter_fcs);
 814         if (rq_attr->modify_bitmask & MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD)
 815                 MLX5_SET(rqc, rq_ctx, vsd, rq_attr->vsd);
 816         if (rq_attr->modify_bitmask &
 817                         MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID)
 818                 MLX5_SET(rqc, rq_ctx, counter_set_id, rq_attr->counter_set_id);
 819         MLX5_SET(rqc, rq_ctx, hairpin_peer_sq, rq_attr->hairpin_peer_sq);
 820         MLX5_SET(rqc, rq_ctx, hairpin_peer_vhca, rq_attr->hairpin_peer_vhca);
 821         if (rq_attr->modify_bitmask & MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_WQ_LWM) {
 822                 wq_ctx = MLX5_ADDR_OF(rqc, rq_ctx, wq);
 823                 MLX5_SET(wq, wq_ctx, lwm, rq_attr->lwm);
 824         }
 825         ret = mlx5_glue->devx_obj_modify(rq->obj, in, sizeof(in),
 826                                          out, sizeof(out));
 827         if (ret) {
 828                 DRV_LOG(ERR, "Failed to modify RQ using DevX");
 829                 rte_errno = errno;
 830                 return -errno;
 831         }
 832         return ret;
 833 }
 834
 835 /**
 836  * Create TIR using DevX API.
 837  *
 838  * @param[in] ctx
 839  *  Context returned from mlx5 open_device() glue function.
 840  * @param [in] tir_attr
 841  *   Pointer to TIR attributes structure.
 842  *
 843  * @return
 844  *   The DevX object created, NULL otherwise and rte_errno is set.
 845  */
 846 struct mlx5_devx_obj *
 847 mlx5_devx_cmd_create_tir(void *ctx,
 848                          struct mlx5_devx_tir_attr *tir_attr)
 849 {
 850         uint32_t in[MLX5_ST_SZ_DW(create_tir_in)] = {0};
 851         uint32_t out[MLX5_ST_SZ_DW(create_tir_out)] = {0};
 852         void *tir_ctx, *outer, *inner, *rss_key;
 853         struct mlx5_devx_obj *tir = NULL;
 854
 855         tir = rte_calloc(__func__, 1, sizeof(*tir), 0);
 856         if (!tir) {
 857                 DRV_LOG(ERR, "Failed to allocate TIR data");
 858                 rte_errno = ENOMEM;
 859                 return NULL;
 860         }
 861         MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
 862         tir_ctx = MLX5_ADDR_OF(create_tir_in, in, ctx);
 863         MLX5_SET(tirc, tir_ctx, disp_type, tir_attr->disp_type);
 864         MLX5_SET(tirc, tir_ctx, lro_timeout_period_usecs,
 865                  tir_attr->lro_timeout_period_usecs);
 866         MLX5_SET(tirc, tir_ctx, lro_enable_mask, tir_attr->lro_enable_mask);
 867         MLX5_SET(tirc, tir_ctx, lro_max_msg_sz, tir_attr->lro_max_msg_sz);
 868         MLX5_SET(tirc, tir_ctx, inline_rqn, tir_attr->inline_rqn);
 869         MLX5_SET(tirc, tir_ctx, rx_hash_symmetric, tir_attr->rx_hash_symmetric);
 870         MLX5_SET(tirc, tir_ctx, tunneled_offload_en,
 871                  tir_attr->tunneled_offload_en);
 872         MLX5_SET(tirc, tir_ctx, indirect_table, tir_attr->indirect_table);
 873         MLX5_SET(tirc, tir_ctx, rx_hash_fn, tir_attr->rx_hash_fn);
 874         MLX5_SET(tirc, tir_ctx, self_lb_block, tir_attr->self_lb_block);
 875         MLX5_SET(tirc, tir_ctx, transport_domain, tir_attr->transport_domain);
 876         rss_key = MLX5_ADDR_OF(tirc, tir_ctx, rx_hash_toeplitz_key);
 877         memcpy(rss_key, tir_attr->rx_hash_toeplitz_key, MLX5_RSS_HASH_KEY_LEN);
 878         outer = MLX5_ADDR_OF(tirc, tir_ctx, rx_hash_field_selector_outer);
 879         MLX5_SET(rx_hash_field_select, outer, l3_prot_type,
 880                  tir_attr->rx_hash_field_selector_outer.l3_prot_type);
 881         MLX5_SET(rx_hash_field_select, outer, l4_prot_type,
 882                  tir_attr->rx_hash_field_selector_outer.l4_prot_type);
 883         MLX5_SET(rx_hash_field_select, outer, selected_fields,
 884                  tir_attr->rx_hash_field_selector_outer.selected_fields);
 885         inner = MLX5_ADDR_OF(tirc, tir_ctx, rx_hash_field_selector_inner);
 886         MLX5_SET(rx_hash_field_select, inner, l3_prot_type,
 887                  tir_attr->rx_hash_field_selector_inner.l3_prot_type);
 888         MLX5_SET(rx_hash_field_select, inner, l4_prot_type,
 889                  tir_attr->rx_hash_field_selector_inner.l4_prot_type);
 890         MLX5_SET(rx_hash_field_select, inner, selected_fields,
 891                  tir_attr->rx_hash_field_selector_inner.selected_fields);
 892         tir->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
 893                                                    out, sizeof(out));
 894         if (!tir->obj) {
 895                 DRV_LOG(ERR, "Failed to create TIR using DevX");
 896                 rte_errno = errno;
 897                 rte_free(tir);
 898                 return NULL;
 899         }
 900         tir->id = MLX5_GET(create_tir_out, out, tirn);
 901         return tir;
 902 }
 903
 904 /**
 905  * Create RQT using DevX API.
 906  *
 907  * @param[in] ctx
 908  *   Context returned from mlx5 open_device() glue function.
 909  * @param [in] rqt_attr
 910  *   Pointer to RQT attributes structure.
 911  *
 912  * @return
 913  *   The DevX object created, NULL otherwise and rte_errno is set.
 914  */
 915 struct mlx5_devx_obj *
 916 mlx5_devx_cmd_create_rqt(void *ctx,
 917                          struct mlx5_devx_rqt_attr *rqt_attr)
 918 {
 919         uint32_t *in = NULL;
 920         uint32_t inlen = MLX5_ST_SZ_BYTES(create_rqt_in) +
 921                          rqt_attr->rqt_actual_size * sizeof(uint32_t);
 922         uint32_t out[MLX5_ST_SZ_DW(create_rqt_out)] = {0};
 923         void *rqt_ctx;
 924         struct mlx5_devx_obj *rqt = NULL;
 925         int i;
 926
 927         in = rte_calloc(__func__, 1, inlen, 0);
 928         if (!in) {
 929                 DRV_LOG(ERR, "Failed to allocate RQT IN data");
 930                 rte_errno = ENOMEM;
 931                 return NULL;
 932         }
 933         rqt = rte_calloc(__func__, 1, sizeof(*rqt), 0);
 934         if (!rqt) {
 935                 DRV_LOG(ERR, "Failed to allocate RQT data");
 936                 rte_errno = ENOMEM;
 937                 rte_free(in);
 938                 return NULL;
 939         }
 940         MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
 941         rqt_ctx = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
 942         MLX5_SET(rqtc, rqt_ctx, list_q_type, rqt_attr->rq_type);
 943         MLX5_SET(rqtc, rqt_ctx, rqt_max_size, rqt_attr->rqt_max_size);
 944         MLX5_SET(rqtc, rqt_ctx, rqt_actual_size, rqt_attr->rqt_actual_size);
 945         for (i = 0; i < rqt_attr->rqt_actual_size; i++)
 946                 MLX5_SET(rqtc, rqt_ctx, rq_num[i], rqt_attr->rq_list[i]);
 947         rqt->obj = mlx5_glue->devx_obj_create(ctx, in, inlen, out, sizeof(out));
 948         rte_free(in);
 949         if (!rqt->obj) {
 950                 DRV_LOG(ERR, "Failed to create RQT using DevX");
 951                 rte_errno = errno;
 952                 rte_free(rqt);
 953                 return NULL;
 954         }
 955         rqt->id = MLX5_GET(create_rqt_out, out, rqtn);
 956         return rqt;
 957 }
 958
 959 /**
 960  * Modify RQT using DevX API.
 961  *
 962  * @param[in] rqt
 963  *   Pointer to RQT DevX object structure.
 964  * @param [in] rqt_attr
 965  *   Pointer to RQT attributes structure.
 966  *
 967  * @return
 968  *   0 on success, a negative errno value otherwise and rte_errno is set.
 969  */
 970 int
 971 mlx5_devx_cmd_modify_rqt(struct mlx5_devx_obj *rqt,
 972                          struct mlx5_devx_rqt_attr *rqt_attr)
 973 {
 974         uint32_t inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) +
 975                          rqt_attr->rqt_actual_size * sizeof(uint32_t);
 976         uint32_t out[MLX5_ST_SZ_DW(modify_rqt_out)] = {0};
 977         uint32_t *in = rte_calloc(__func__, 1, inlen, 0);
 978         void *rqt_ctx;
 979         int i;
 980         int ret;
 981
 982         if (!in) {
 983                 DRV_LOG(ERR, "Failed to allocate RQT modify IN data.");
 984                 rte_errno = ENOMEM;
 985                 return -ENOMEM;
 986         }
 987         MLX5_SET(modify_rqt_in, in, opcode, MLX5_CMD_OP_MODIFY_RQT);
 988         MLX5_SET(modify_rqt_in, in, rqtn, rqt->id);
 989         MLX5_SET64(modify_rqt_in, in, modify_bitmask, 0x1);
 990         rqt_ctx = MLX5_ADDR_OF(modify_rqt_in, in, rqt_context);
 991         MLX5_SET(rqtc, rqt_ctx, list_q_type, rqt_attr->rq_type);
 992         MLX5_SET(rqtc, rqt_ctx, rqt_max_size, rqt_attr->rqt_max_size);
 993         MLX5_SET(rqtc, rqt_ctx, rqt_actual_size, rqt_attr->rqt_actual_size);
 994         for (i = 0; i < rqt_attr->rqt_actual_size; i++)
 995                 MLX5_SET(rqtc, rqt_ctx, rq_num[i], rqt_attr->rq_list[i]);
 996         ret = mlx5_glue->devx_obj_modify(rqt->obj, in, inlen, out, sizeof(out));
 997         rte_free(in);
 998         if (ret) {
 999                 DRV_LOG(ERR, "Failed to modify RQT using DevX.");
1000                 rte_errno = errno;
1001                 return -rte_errno;
1002         }
1003         return ret;
1004 }
1005
1006 /**
1007  * Create SQ using DevX API.
1008  *
1009  * @param[in] ctx
1010  *   Context returned from mlx5 open_device() glue function.
1011  * @param [in] sq_attr
1012  *   Pointer to SQ attributes structure.
1013  * @param [in] socket
1014  *   CPU socket ID for allocations.
1015  *
1016  * @return
1017  *   The DevX object created, NULL otherwise and rte_errno is set.
1018  **/
1019 struct mlx5_devx_obj *
1020 mlx5_devx_cmd_create_sq(void *ctx,
1021                         struct mlx5_devx_create_sq_attr *sq_attr)
1022 {
1023         uint32_t in[MLX5_ST_SZ_DW(create_sq_in)] = {0};
1024         uint32_t out[MLX5_ST_SZ_DW(create_sq_out)] = {0};
1025         void *sq_ctx;
1026         void *wq_ctx;
1027         struct mlx5_devx_wq_attr *wq_attr;
1028         struct mlx5_devx_obj *sq = NULL;
1029
1030         sq = rte_calloc(__func__, 1, sizeof(*sq), 0);
1031         if (!sq) {
1032                 DRV_LOG(ERR, "Failed to allocate SQ data");
1033                 rte_errno = ENOMEM;
1034                 return NULL;
1035         }
1036         MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
1037         sq_ctx = MLX5_ADDR_OF(create_sq_in, in, ctx);
1038         MLX5_SET(sqc, sq_ctx, rlky, sq_attr->rlky);
1039         MLX5_SET(sqc, sq_ctx, cd_master, sq_attr->cd_master);
1040         MLX5_SET(sqc, sq_ctx, fre, sq_attr->fre);
1041         MLX5_SET(sqc, sq_ctx, flush_in_error_en, sq_attr->flush_in_error_en);
1042         MLX5_SET(sqc, sq_ctx, allow_multi_pkt_send_wqe,
1043                  sq_attr->flush_in_error_en);
1044         MLX5_SET(sqc, sq_ctx, min_wqe_inline_mode,
1045                  sq_attr->min_wqe_inline_mode);
1046         MLX5_SET(sqc, sq_ctx, state, sq_attr->state);
1047         MLX5_SET(sqc, sq_ctx, reg_umr, sq_attr->reg_umr);
1048         MLX5_SET(sqc, sq_ctx, allow_swp, sq_attr->allow_swp);
1049         MLX5_SET(sqc, sq_ctx, hairpin, sq_attr->hairpin);
1050         MLX5_SET(sqc, sq_ctx, non_wire, sq_attr->non_wire);
1051         MLX5_SET(sqc, sq_ctx, static_sq_wq, sq_attr->static_sq_wq);
1052         MLX5_SET(sqc, sq_ctx, user_index, sq_attr->user_index);
1053         MLX5_SET(sqc, sq_ctx, cqn, sq_attr->cqn);
1054         MLX5_SET(sqc, sq_ctx, packet_pacing_rate_limit_index,
1055                  sq_attr->packet_pacing_rate_limit_index);
1056         MLX5_SET(sqc, sq_ctx, tis_lst_sz, sq_attr->tis_lst_sz);
1057         MLX5_SET(sqc, sq_ctx, tis_num_0, sq_attr->tis_num);
1058         wq_ctx = MLX5_ADDR_OF(sqc, sq_ctx, wq);
1059         wq_attr = &sq_attr->wq_attr;
1060         devx_cmd_fill_wq_data(wq_ctx, wq_attr);
1061         sq->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
1062                                              out, sizeof(out));
1063         if (!sq->obj) {
1064                 DRV_LOG(ERR, "Failed to create SQ using DevX");
1065                 rte_errno = errno;
1066                 rte_free(sq);
1067                 return NULL;
1068         }
1069         sq->id = MLX5_GET(create_sq_out, out, sqn);
1070         return sq;
1071 }
1072
1073 /**
1074  * Modify SQ using DevX API.
1075  *
1076  * @param[in] sq
1077  *   Pointer to SQ object structure.
1078  * @param [in] sq_attr
1079  *   Pointer to SQ attributes structure.
1080  *
1081  * @return
1082  *   0 on success, a negative errno value otherwise and rte_errno is set.
1083  */
1084 int
1085 mlx5_devx_cmd_modify_sq(struct mlx5_devx_obj *sq,
1086                         struct mlx5_devx_modify_sq_attr *sq_attr)
1087 {
1088         uint32_t in[MLX5_ST_SZ_DW(modify_sq_in)] = {0};
1089         uint32_t out[MLX5_ST_SZ_DW(modify_sq_out)] = {0};
1090         void *sq_ctx;
1091         int ret;
1092
1093         MLX5_SET(modify_sq_in, in, opcode, MLX5_CMD_OP_MODIFY_SQ);
1094         MLX5_SET(modify_sq_in, in, sq_state, sq_attr->sq_state);
1095         MLX5_SET(modify_sq_in, in, sqn, sq->id);
1096         sq_ctx = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1097         MLX5_SET(sqc, sq_ctx, state, sq_attr->state);
1098         MLX5_SET(sqc, sq_ctx, hairpin_peer_rq, sq_attr->hairpin_peer_rq);
1099         MLX5_SET(sqc, sq_ctx, hairpin_peer_vhca, sq_attr->hairpin_peer_vhca);
1100         ret = mlx5_glue->devx_obj_modify(sq->obj, in, sizeof(in),
1101                                          out, sizeof(out));
1102         if (ret) {
1103                 DRV_LOG(ERR, "Failed to modify SQ using DevX");
1104                 rte_errno = errno;
1105                 return -errno;
1106         }
1107         return ret;
1108 }
1109
1110 /**
1111  * Create TIS using DevX API.
1112  *
1113  * @param[in] ctx
1114  *   Context returned from mlx5 open_device() glue function.
1115  * @param [in] tis_attr
1116  *   Pointer to TIS attributes structure.
1117  *
1118  * @return
1119  *   The DevX object created, NULL otherwise and rte_errno is set.
1120  */
1121 struct mlx5_devx_obj *
1122 mlx5_devx_cmd_create_tis(void *ctx,
1123                          struct mlx5_devx_tis_attr *tis_attr)
1124 {
1125         uint32_t in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
1126         uint32_t out[MLX5_ST_SZ_DW(create_tis_out)] = {0};
1127         struct mlx5_devx_obj *tis = NULL;
1128         void *tis_ctx;
1129
1130         tis = rte_calloc(__func__, 1, sizeof(*tis), 0);
1131         if (!tis) {
1132                 DRV_LOG(ERR, "Failed to allocate TIS object");
1133                 rte_errno = ENOMEM;
1134                 return NULL;
1135         }
1136         MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS);
1137         tis_ctx = MLX5_ADDR_OF(create_tis_in, in, ctx);
1138         MLX5_SET(tisc, tis_ctx, strict_lag_tx_port_affinity,
1139                  tis_attr->strict_lag_tx_port_affinity);
1140         MLX5_SET(tisc, tis_ctx, strict_lag_tx_port_affinity,
1141                  tis_attr->strict_lag_tx_port_affinity);
1142         MLX5_SET(tisc, tis_ctx, prio, tis_attr->prio);
1143         MLX5_SET(tisc, tis_ctx, transport_domain,
1144                  tis_attr->transport_domain);
1145         tis->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
1146                                               out, sizeof(out));
1147         if (!tis->obj) {
1148                 DRV_LOG(ERR, "Failed to create TIS using DevX");
1149                 rte_errno = errno;
1150                 rte_free(tis);
1151                 return NULL;
1152         }
1153         tis->id = MLX5_GET(create_tis_out, out, tisn);
1154         return tis;
1155 }
1156
1157 /**
1158  * Create transport domain using DevX API.
1159  *
1160  * @param[in] ctx
1161  *   Context returned from mlx5 open_device() glue function.
1162  * @return
1163  *   The DevX object created, NULL otherwise and rte_errno is set.
1164  */
1165 struct mlx5_devx_obj *
1166 mlx5_devx_cmd_create_td(void *ctx)
1167 {
1168         uint32_t in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {0};
1169         uint32_t out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {0};
1170         struct mlx5_devx_obj *td = NULL;
1171
1172         td = rte_calloc(__func__, 1, sizeof(*td), 0);
1173         if (!td) {
1174                 DRV_LOG(ERR, "Failed to allocate TD object");
1175                 rte_errno = ENOMEM;
1176                 return NULL;
1177         }
1178         MLX5_SET(alloc_transport_domain_in, in, opcode,
1179                  MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
1180         td->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in),
1181                                              out, sizeof(out));
1182         if (!td->obj) {
1183                 DRV_LOG(ERR, "Failed to create TIS using DevX");
1184                 rte_errno = errno;
1185                 rte_free(td);
1186                 return NULL;
1187         }
1188         td->id = MLX5_GET(alloc_transport_domain_out, out,
1189                            transport_domain);
1190         return td;
1191 }
1192
1193 /**
1194  * Dump all flows to file.
1195  *
1196  * @param[in] fdb_domain
1197  *   FDB domain.
1198  * @param[in] rx_domain
1199  *   RX domain.
1200  * @param[in] tx_domain
1201  *   TX domain.
1202  * @param[out] file
1203  *   Pointer to file stream.
1204  *
1205  * @return
1206  *   0 on success, a nagative value otherwise.
1207  */
1208 int
1209 mlx5_devx_cmd_flow_dump(void *fdb_domain __rte_unused,
1210                         void *rx_domain __rte_unused,
1211                         void *tx_domain __rte_unused, FILE *file __rte_unused)
1212 {
1213         int ret = 0;
1214
1215 #ifdef HAVE_MLX5_DR_FLOW_DUMP
1216         if (fdb_domain) {
1217                 ret = mlx5_glue->dr_dump_domain(file, fdb_domain);
1218                 if (ret)
1219                         return ret;
1220         }
1221         MLX5_ASSERT(rx_domain);
1222         ret = mlx5_glue->dr_dump_domain(file, rx_domain);
1223         if (ret)
1224                 return ret;
1225         MLX5_ASSERT(tx_domain);
1226         ret = mlx5_glue->dr_dump_domain(file, tx_domain);
1227 #else
1228         ret = ENOTSUP;
1229 #endif
1230         return -ret;
1231 }
1232
1233 /*
1234  * Create CQ using DevX API.
1235  *
1236  * @param[in] ctx
1237  *   Context returned from mlx5 open_device() glue function.
1238  * @param [in] attr
1239  *   Pointer to CQ attributes structure.
1240  *
1241  * @return
1242  *   The DevX object created, NULL otherwise and rte_errno is set.
1243  */
1244 struct mlx5_devx_obj *
1245 mlx5_devx_cmd_create_cq(void *ctx, struct mlx5_devx_cq_attr *attr)
1246 {
1247         uint32_t in[MLX5_ST_SZ_DW(create_cq_in)] = {0};
1248         uint32_t out[MLX5_ST_SZ_DW(create_cq_out)] = {0};
1249         struct mlx5_devx_obj *cq_obj = rte_zmalloc(__func__, sizeof(*cq_obj),
1250                                                    0);
1251         void *cqctx = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1252
1253         if (!cq_obj) {
1254                 DRV_LOG(ERR, "Failed to allocate CQ object memory.");
1255                 rte_errno = ENOMEM;
1256                 return NULL;
1257         }
1258         MLX5_SET(create_cq_in, in, opcode, MLX5_CMD_OP_CREATE_CQ);
1259         if (attr->db_umem_valid) {
1260                 MLX5_SET(cqc, cqctx, dbr_umem_valid, attr->db_umem_valid);
1261                 MLX5_SET(cqc, cqctx, dbr_umem_id, attr->db_umem_id);
1262                 MLX5_SET64(cqc, cqctx, dbr_addr, attr->db_umem_offset);
1263         } else {
1264                 MLX5_SET64(cqc, cqctx, dbr_addr, attr->db_addr);
1265         }
1266         MLX5_SET(cqc, cqctx, cqe_sz, attr->cqe_size);
1267         MLX5_SET(cqc, cqctx, cc, attr->use_first_only);
1268         MLX5_SET(cqc, cqctx, oi, attr->overrun_ignore);
1269         MLX5_SET(cqc, cqctx, log_cq_size, attr->log_cq_size);
1270         MLX5_SET(cqc, cqctx, log_page_size, attr->log_page_size -
1271                  MLX5_ADAPTER_PAGE_SHIFT);
1272         MLX5_SET(cqc, cqctx, c_eqn, attr->eqn);
1273         MLX5_SET(cqc, cqctx, uar_page, attr->uar_page_id);
1274         if (attr->q_umem_valid) {
1275                 MLX5_SET(create_cq_in, in, cq_umem_valid, attr->q_umem_valid);
1276                 MLX5_SET(create_cq_in, in, cq_umem_id, attr->q_umem_id);
1277                 MLX5_SET64(create_cq_in, in, cq_umem_offset,
1278                            attr->q_umem_offset);
1279         }
1280         cq_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out,
1281                                                  sizeof(out));
1282         if (!cq_obj->obj) {
1283                 rte_errno = errno;
1284                 DRV_LOG(ERR, "Failed to create CQ using DevX errno=%d.", errno);
1285                 rte_free(cq_obj);
1286                 return NULL;
1287         }
1288         cq_obj->id = MLX5_GET(create_cq_out, out, cqn);
1289         return cq_obj;
1290 }
1291
1292 /**
1293  * Create VIRTQ using DevX API.
1294  *
1295  * @param[in] ctx
1296  *   Context returned from mlx5 open_device() glue function.
1297  * @param [in] attr
1298  *   Pointer to VIRTQ attributes structure.
1299  *
1300  * @return
1301  *   The DevX object created, NULL otherwise and rte_errno is set.
1302  */
1303 struct mlx5_devx_obj *
1304 mlx5_devx_cmd_create_virtq(void *ctx,
1305                            struct mlx5_devx_virtq_attr *attr)
1306 {
1307         uint32_t in[MLX5_ST_SZ_DW(create_virtq_in)] = {0};
1308         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
1309         struct mlx5_devx_obj *virtq_obj = rte_zmalloc(__func__,
1310                                                      sizeof(*virtq_obj), 0);
1311         void *virtq = MLX5_ADDR_OF(create_virtq_in, in, virtq);
1312         void *hdr = MLX5_ADDR_OF(create_virtq_in, in, hdr);
1313         void *virtctx = MLX5_ADDR_OF(virtio_net_q, virtq, virtio_q_context);
1314
1315         if (!virtq_obj) {
1316                 DRV_LOG(ERR, "Failed to allocate virtq data.");
1317                 rte_errno = ENOMEM;
1318                 return NULL;
1319         }
1320         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
1321                  MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
1322         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
1323                  MLX5_GENERAL_OBJ_TYPE_VIRTQ);
1324         MLX5_SET16(virtio_net_q, virtq, hw_available_index,
1325                    attr->hw_available_index);
1326         MLX5_SET16(virtio_net_q, virtq, hw_used_index, attr->hw_used_index);
1327         MLX5_SET16(virtio_net_q, virtq, tso_ipv4, attr->tso_ipv4);
1328         MLX5_SET16(virtio_net_q, virtq, tso_ipv6, attr->tso_ipv6);
1329         MLX5_SET16(virtio_net_q, virtq, tx_csum, attr->tx_csum);
1330         MLX5_SET16(virtio_net_q, virtq, rx_csum, attr->rx_csum);
1331         MLX5_SET16(virtio_q, virtctx, virtio_version_1_0,
1332                    attr->virtio_version_1_0);
1333         MLX5_SET16(virtio_q, virtctx, event_mode, attr->event_mode);
1334         MLX5_SET(virtio_q, virtctx, event_qpn_or_msix, attr->qp_id);
1335         MLX5_SET64(virtio_q, virtctx, desc_addr, attr->desc_addr);
1336         MLX5_SET64(virtio_q, virtctx, used_addr, attr->used_addr);
1337         MLX5_SET64(virtio_q, virtctx, available_addr, attr->available_addr);
1338         MLX5_SET16(virtio_q, virtctx, queue_index, attr->queue_index);
1339         MLX5_SET16(virtio_q, virtctx, queue_size, attr->q_size);
1340         MLX5_SET(virtio_q, virtctx, virtio_q_mkey, attr->mkey);
1341         MLX5_SET(virtio_q, virtctx, umem_1_id, attr->umems[0].id);
1342         MLX5_SET(virtio_q, virtctx, umem_1_size, attr->umems[0].size);
1343         MLX5_SET64(virtio_q, virtctx, umem_1_offset, attr->umems[0].offset);
1344         MLX5_SET(virtio_q, virtctx, umem_2_id, attr->umems[1].id);
1345         MLX5_SET(virtio_q, virtctx, umem_2_size, attr->umems[1].size);
1346         MLX5_SET64(virtio_q, virtctx, umem_2_offset, attr->umems[1].offset);
1347         MLX5_SET(virtio_q, virtctx, umem_3_id, attr->umems[2].id);
1348         MLX5_SET(virtio_q, virtctx, umem_3_size, attr->umems[2].size);
1349         MLX5_SET64(virtio_q, virtctx, umem_3_offset, attr->umems[2].offset);
1350         MLX5_SET(virtio_q, virtctx, counter_set_id, attr->counters_obj_id);
1351         MLX5_SET(virtio_q, virtctx, pd, attr->pd);
1352         MLX5_SET(virtio_net_q, virtq, tisn_or_qpn, attr->tis_id);
1353         virtq_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out,
1354                                                     sizeof(out));
1355         if (!virtq_obj->obj) {
1356                 rte_errno = errno;
1357                 DRV_LOG(ERR, "Failed to create VIRTQ Obj using DevX.");
1358                 rte_free(virtq_obj);
1359                 return NULL;
1360         }
1361         virtq_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
1362         return virtq_obj;
1363 }
1364
1365 /**
1366  * Modify VIRTQ using DevX API.
1367  *
1368  * @param[in] virtq_obj
1369  *   Pointer to virtq object structure.
1370  * @param [in] attr
1371  *   Pointer to modify virtq attributes structure.
1372  *
1373  * @return
1374  *   0 on success, a negative errno value otherwise and rte_errno is set.
1375  */
1376 int
1377 mlx5_devx_cmd_modify_virtq(struct mlx5_devx_obj *virtq_obj,
1378                            struct mlx5_devx_virtq_attr *attr)
1379 {
1380         uint32_t in[MLX5_ST_SZ_DW(create_virtq_in)] = {0};
1381         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
1382         void *virtq = MLX5_ADDR_OF(create_virtq_in, in, virtq);
1383         void *hdr = MLX5_ADDR_OF(create_virtq_in, in, hdr);
1384         void *virtctx = MLX5_ADDR_OF(virtio_net_q, virtq, virtio_q_context);
1385         int ret;
1386
1387         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
1388                  MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
1389         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
1390                  MLX5_GENERAL_OBJ_TYPE_VIRTQ);
1391         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_id, virtq_obj->id);
1392         MLX5_SET64(virtio_net_q, virtq, modify_field_select, attr->type);
1393         MLX5_SET16(virtio_q, virtctx, queue_index, attr->queue_index);
1394         switch (attr->type) {
1395         case MLX5_VIRTQ_MODIFY_TYPE_STATE:
1396                 MLX5_SET16(virtio_net_q, virtq, state, attr->state);
1397                 break;
1398         case MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_PARAMS:
1399                 MLX5_SET(virtio_net_q, virtq, dirty_bitmap_mkey,
1400                          attr->dirty_bitmap_mkey);
1401                 MLX5_SET64(virtio_net_q, virtq, dirty_bitmap_addr,
1402                          attr->dirty_bitmap_addr);
1403                 MLX5_SET(virtio_net_q, virtq, dirty_bitmap_size,
1404                          attr->dirty_bitmap_size);
1405                 break;
1406         case MLX5_VIRTQ_MODIFY_TYPE_DIRTY_BITMAP_DUMP_ENABLE:
1407                 MLX5_SET(virtio_net_q, virtq, dirty_bitmap_dump_enable,
1408                          attr->dirty_bitmap_dump_enable);
1409                 break;
1410         default:
1411                 rte_errno = EINVAL;
1412                 return -rte_errno;
1413         }
1414         ret = mlx5_glue->devx_obj_modify(virtq_obj->obj, in, sizeof(in),
1415                                          out, sizeof(out));
1416         if (ret) {
1417                 DRV_LOG(ERR, "Failed to modify VIRTQ using DevX.");
1418                 rte_errno = errno;
1419                 return -errno;
1420         }
1421         return ret;
1422 }
1423
1424 /**
1425  * Query VIRTQ using DevX API.
1426  *
1427  * @param[in] virtq_obj
1428  *   Pointer to virtq object structure.
1429  * @param [in/out] attr
1430  *   Pointer to virtq attributes structure.
1431  *
1432  * @return
1433  *   0 on success, a negative errno value otherwise and rte_errno is set.
1434  */
1435 int
1436 mlx5_devx_cmd_query_virtq(struct mlx5_devx_obj *virtq_obj,
1437                            struct mlx5_devx_virtq_attr *attr)
1438 {
1439         uint32_t in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {0};
1440         uint32_t out[MLX5_ST_SZ_DW(query_virtq_out)] = {0};
1441         void *hdr = MLX5_ADDR_OF(query_virtq_out, in, hdr);
1442         void *virtq = MLX5_ADDR_OF(query_virtq_out, out, virtq);
1443         int ret;
1444
1445         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
1446                  MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
1447         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
1448                  MLX5_GENERAL_OBJ_TYPE_VIRTQ);
1449         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_id, virtq_obj->id);
1450         ret = mlx5_glue->devx_obj_query(virtq_obj->obj, in, sizeof(in),
1451                                          out, sizeof(out));
1452         if (ret) {
1453                 DRV_LOG(ERR, "Failed to modify VIRTQ using DevX.");
1454                 rte_errno = errno;
1455                 return -errno;
1456         }
1457         attr->hw_available_index = MLX5_GET16(virtio_net_q, virtq,
1458                                               hw_available_index);
1459         attr->hw_used_index = MLX5_GET16(virtio_net_q, virtq, hw_used_index);
1460         return ret;
1461 }
1462
1463 /**
1464  * Create QP using DevX API.
1465  *
1466  * @param[in] ctx
1467  *   Context returned from mlx5 open_device() glue function.
1468  * @param [in] attr
1469  *   Pointer to QP attributes structure.
1470  *
1471  * @return
1472  *   The DevX object created, NULL otherwise and rte_errno is set.
1473  */
1474 struct mlx5_devx_obj *
1475 mlx5_devx_cmd_create_qp(void *ctx,
1476                         struct mlx5_devx_qp_attr *attr)
1477 {
1478         uint32_t in[MLX5_ST_SZ_DW(create_qp_in)] = {0};
1479         uint32_t out[MLX5_ST_SZ_DW(create_qp_out)] = {0};
1480         struct mlx5_devx_obj *qp_obj = rte_zmalloc(__func__, sizeof(*qp_obj),
1481                                                    0);
1482         void *qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
1483
1484         if (!qp_obj) {
1485                 DRV_LOG(ERR, "Failed to allocate QP data.");
1486                 rte_errno = ENOMEM;
1487                 return NULL;
1488         }
1489         MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
1490         MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
1491         MLX5_SET(qpc, qpc, pd, attr->pd);
1492         if (attr->uar_index) {
1493                 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
1494                 MLX5_SET(qpc, qpc, uar_page, attr->uar_index);
1495                 MLX5_SET(qpc, qpc, log_page_size, attr->log_page_size -
1496                          MLX5_ADAPTER_PAGE_SHIFT);
1497                 if (attr->sq_size) {
1498                         MLX5_ASSERT(RTE_IS_POWER_OF_2(attr->sq_size));
1499                         MLX5_SET(qpc, qpc, cqn_snd, attr->cqn);
1500                         MLX5_SET(qpc, qpc, log_sq_size,
1501                                  rte_log2_u32(attr->sq_size));
1502                 } else {
1503                         MLX5_SET(qpc, qpc, no_sq, 1);
1504                 }
1505                 if (attr->rq_size) {
1506                         MLX5_ASSERT(RTE_IS_POWER_OF_2(attr->rq_size));
1507                         MLX5_SET(qpc, qpc, cqn_rcv, attr->cqn);
1508                         MLX5_SET(qpc, qpc, log_rq_stride, attr->log_rq_stride -
1509                                  MLX5_LOG_RQ_STRIDE_SHIFT);
1510                         MLX5_SET(qpc, qpc, log_rq_size,
1511                                  rte_log2_u32(attr->rq_size));
1512                         MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ);
1513                 } else {
1514                         MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ);
1515                 }
1516                 if (attr->dbr_umem_valid) {
1517                         MLX5_SET(qpc, qpc, dbr_umem_valid,
1518                                  attr->dbr_umem_valid);
1519                         MLX5_SET(qpc, qpc, dbr_umem_id, attr->dbr_umem_id);
1520                 }
1521                 MLX5_SET64(qpc, qpc, dbr_addr, attr->dbr_address);
1522                 MLX5_SET64(create_qp_in, in, wq_umem_offset,
1523                            attr->wq_umem_offset);
1524                 MLX5_SET(create_qp_in, in, wq_umem_id, attr->wq_umem_id);
1525                 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
1526         } else {
1527                 /* Special QP to be managed by FW - no SQ\RQ\CQ\UAR\DB rec. */
1528                 MLX5_SET(qpc, qpc, rq_type, MLX5_ZERO_LEN_RQ);
1529                 MLX5_SET(qpc, qpc, no_sq, 1);
1530         }
1531         qp_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out,
1532                                                  sizeof(out));
1533         if (!qp_obj->obj) {
1534                 rte_errno = errno;
1535                 DRV_LOG(ERR, "Failed to create QP Obj using DevX.");
1536                 rte_free(qp_obj);
1537                 return NULL;
1538         }
1539         qp_obj->id = MLX5_GET(create_qp_out, out, qpn);
1540         return qp_obj;
1541 }
1542
1543 /**
1544  * Modify QP using DevX API.
1545  * Currently supports only force loop-back QP.
1546  *
1547  * @param[in] qp
1548  *   Pointer to QP object structure.
1549  * @param [in] qp_st_mod_op
1550  *   The QP state modification operation.
1551  * @param [in] remote_qp_id
1552  *   The remote QP ID for MLX5_CMD_OP_INIT2RTR_QP operation.
1553  *
1554  * @return
1555  *   0 on success, a negative errno value otherwise and rte_errno is set.
1556  */
1557 int
1558 mlx5_devx_cmd_modify_qp_state(struct mlx5_devx_obj *qp, uint32_t qp_st_mod_op,
1559                               uint32_t remote_qp_id)
1560 {
1561         union {
1562                 uint32_t rst2init[MLX5_ST_SZ_DW(rst2init_qp_in)];
1563                 uint32_t init2rtr[MLX5_ST_SZ_DW(init2rtr_qp_in)];
1564                 uint32_t rtr2rts[MLX5_ST_SZ_DW(rtr2rts_qp_in)];
1565         } in;
1566         union {
1567                 uint32_t rst2init[MLX5_ST_SZ_DW(rst2init_qp_out)];
1568                 uint32_t init2rtr[MLX5_ST_SZ_DW(init2rtr_qp_out)];
1569                 uint32_t rtr2rts[MLX5_ST_SZ_DW(rtr2rts_qp_out)];
1570         } out;
1571         void *qpc;
1572         int ret;
1573         unsigned int inlen;
1574         unsigned int outlen;
1575
1576         memset(&in, 0, sizeof(in));
1577         memset(&out, 0, sizeof(out));
1578         MLX5_SET(rst2init_qp_in, &in, opcode, qp_st_mod_op);
1579         switch (qp_st_mod_op) {
1580         case MLX5_CMD_OP_RST2INIT_QP:
1581                 MLX5_SET(rst2init_qp_in, &in, qpn, qp->id);
1582                 qpc = MLX5_ADDR_OF(rst2init_qp_in, &in, qpc);
1583                 MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1);
1584                 MLX5_SET(qpc, qpc, rre, 1);
1585                 MLX5_SET(qpc, qpc, rwe, 1);
1586                 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
1587                 inlen = sizeof(in.rst2init);
1588                 outlen = sizeof(out.rst2init);
1589                 break;
1590         case MLX5_CMD_OP_INIT2RTR_QP:
1591                 MLX5_SET(init2rtr_qp_in, &in, qpn, qp->id);
1592                 qpc = MLX5_ADDR_OF(init2rtr_qp_in, &in, qpc);
1593                 MLX5_SET(qpc, qpc, primary_address_path.fl, 1);
1594                 MLX5_SET(qpc, qpc, primary_address_path.vhca_port_num, 1);
1595                 MLX5_SET(qpc, qpc, mtu, 1);
1596                 MLX5_SET(qpc, qpc, log_msg_max, 30);
1597                 MLX5_SET(qpc, qpc, remote_qpn, remote_qp_id);
1598                 MLX5_SET(qpc, qpc, min_rnr_nak, 0);
1599                 inlen = sizeof(in.init2rtr);
1600                 outlen = sizeof(out.init2rtr);
1601                 break;
1602         case MLX5_CMD_OP_RTR2RTS_QP:
1603                 qpc = MLX5_ADDR_OF(rtr2rts_qp_in, &in, qpc);
1604                 MLX5_SET(rtr2rts_qp_in, &in, qpn, qp->id);
1605                 MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 14);
1606                 MLX5_SET(qpc, qpc, log_ack_req_freq, 0);
1607                 MLX5_SET(qpc, qpc, retry_count, 7);
1608                 MLX5_SET(qpc, qpc, rnr_retry, 7);
1609                 inlen = sizeof(in.rtr2rts);
1610                 outlen = sizeof(out.rtr2rts);
1611                 break;
1612         default:
1613                 DRV_LOG(ERR, "Invalid or unsupported QP modify op %u.",
1614                         qp_st_mod_op);
1615                 rte_errno = EINVAL;
1616                 return -rte_errno;
1617         }
1618         ret = mlx5_glue->devx_obj_modify(qp->obj, &in, inlen, &out, outlen);
1619         if (ret) {
1620                 DRV_LOG(ERR, "Failed to modify QP using DevX.");
1621                 rte_errno = errno;
1622                 return -errno;
1623         }
1624         return ret;
1625 }
1626
1627 struct mlx5_devx_obj *
1628 mlx5_devx_cmd_create_virtio_q_counters(void *ctx)
1629 {
1630         uint32_t in[MLX5_ST_SZ_DW(create_virtio_q_counters_in)] = {0};
1631         uint32_t out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
1632         struct mlx5_devx_obj *couners_obj = rte_zmalloc(__func__,
1633                                                        sizeof(*couners_obj), 0);
1634         void *hdr = MLX5_ADDR_OF(create_virtio_q_counters_in, in, hdr);
1635
1636         if (!couners_obj) {
1637                 DRV_LOG(ERR, "Failed to allocate virtio queue counters data.");
1638                 rte_errno = ENOMEM;
1639                 return NULL;
1640         }
1641         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
1642                  MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
1643         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
1644                  MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS);
1645         couners_obj->obj = mlx5_glue->devx_obj_create(ctx, in, sizeof(in), out,
1646                                                       sizeof(out));
1647         if (!couners_obj->obj) {
1648                 rte_errno = errno;
1649                 DRV_LOG(ERR, "Failed to create virtio queue counters Obj using"
1650                         " DevX.");
1651                 rte_free(couners_obj);
1652                 return NULL;
1653         }
1654         couners_obj->id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
1655         return couners_obj;
1656 }
1657
1658 int
1659 mlx5_devx_cmd_query_virtio_q_counters(struct mlx5_devx_obj *couners_obj,
1660                                    struct mlx5_devx_virtio_q_couners_attr *attr)
1661 {
1662         uint32_t in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {0};
1663         uint32_t out[MLX5_ST_SZ_DW(query_virtio_q_counters_out)] = {0};
1664         void *hdr = MLX5_ADDR_OF(query_virtio_q_counters_out, in, hdr);
1665         void *virtio_q_counters = MLX5_ADDR_OF(query_virtio_q_counters_out, out,
1666                                                virtio_q_counters);
1667         int ret;
1668
1669         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode,
1670                  MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
1671         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type,
1672                  MLX5_GENERAL_OBJ_TYPE_VIRTIO_Q_COUNTERS);
1673         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_id, couners_obj->id);
1674         ret = mlx5_glue->devx_obj_query(couners_obj->obj, in, sizeof(in), out,
1675                                         sizeof(out));
1676         if (ret) {
1677                 DRV_LOG(ERR, "Failed to query virtio q counters using DevX.");
1678                 rte_errno = errno;
1679                 return -errno;
1680         }
1681         attr->received_desc = MLX5_GET64(virtio_q_counters, virtio_q_counters,
1682                                          received_desc);
1683         attr->completed_desc = MLX5_GET64(virtio_q_counters, virtio_q_counters,
1684                                           completed_desc);
1685         attr->error_cqes = MLX5_GET(virtio_q_counters, virtio_q_counters,
1686                                     error_cqes);
1687         attr->bad_desc_errors = MLX5_GET(virtio_q_counters, virtio_q_counters,
1688                                          bad_desc_errors);
1689         attr->exceed_max_chain = MLX5_GET(virtio_q_counters, virtio_q_counters,
1690                                           exceed_max_chain);
1691         attr->invalid_buffer = MLX5_GET(virtio_q_counters, virtio_q_counters,
1692                                         invalid_buffer);
1693         return ret;
1694 }